From 4b4821888e3a8ab67a524c11af8854936e824f8a Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Oct 2023 15:55:58 +1300 Subject: [PATCH 1/4] Add update script for a new vendor directory Where we'll put pyproject.nix --- vendor/update.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 vendor/update.py diff --git a/vendor/update.py b/vendor/update.py new file mode 100755 index 0000000..5e35911 --- /dev/null +++ b/vendor/update.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import subprocess +import shutil +import json +import os + + +if __name__ == "__main__": + store_path = json.loads( + subprocess.check_output( + [ + "nix-instantiate", + "--eval", + "--json", + "--expr", + 'builtins.fetchGit { url = "git@github.com:adisbladis/pyproject.nix.git"; }', + ] + ) + ) + + try: + shutil.rmtree("pyproject.nix") + except FileNotFoundError: + pass + + os.mkdir("pyproject.nix") + + for filename in os.listdir(f"{store_path}/lib"): + if filename.startswith("test") or not filename.endswith(".nix"): + continue + shutil.copy(f"{store_path}/lib/{filename}", f"pyproject.nix/{filename}") From c061e4c46942b268130d7979689215e14a15987f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Oct 2023 15:57:38 +1300 Subject: [PATCH 2/4] Add vendored copy of pyproject.nix This will be used for a lot of primitives currently implemented in poetry2nix. --- vendor/pyproject.nix/default.nix | 22 ++ vendor/pyproject.nix/pep427.nix | 49 +++ vendor/pyproject.nix/pep440.nix | 261 +++++++++++++ vendor/pyproject.nix/pep508.nix | 544 ++++++++++++++++++++++++++++ vendor/pyproject.nix/pep518.nix | 12 + vendor/pyproject.nix/pep599.nix | 20 + vendor/pyproject.nix/pep600.nix | 58 +++ vendor/pyproject.nix/pep621.nix | 147 ++++++++ vendor/pyproject.nix/pip.nix | 105 ++++++ vendor/pyproject.nix/poetry.nix | 249 +++++++++++++ vendor/pyproject.nix/project.nix | 82 +++++ vendor/pyproject.nix/pypa.nix | 22 ++ vendor/pyproject.nix/renderers.nix | 147 ++++++++ vendor/pyproject.nix/util.nix | 11 + vendor/pyproject.nix/validators.nix | 71 ++++ 15 files changed, 1800 insertions(+) create mode 100644 vendor/pyproject.nix/default.nix create mode 100644 vendor/pyproject.nix/pep427.nix create mode 100644 vendor/pyproject.nix/pep440.nix create mode 100644 vendor/pyproject.nix/pep508.nix create mode 100644 vendor/pyproject.nix/pep518.nix create mode 100644 vendor/pyproject.nix/pep599.nix create mode 100644 vendor/pyproject.nix/pep600.nix create mode 100644 vendor/pyproject.nix/pep621.nix create mode 100644 vendor/pyproject.nix/pip.nix create mode 100644 vendor/pyproject.nix/poetry.nix create mode 100644 vendor/pyproject.nix/project.nix create mode 100644 vendor/pyproject.nix/pypa.nix create mode 100644 vendor/pyproject.nix/renderers.nix create mode 100644 vendor/pyproject.nix/util.nix create mode 100644 vendor/pyproject.nix/validators.nix diff --git a/vendor/pyproject.nix/default.nix b/vendor/pyproject.nix/default.nix new file mode 100644 index 0000000..bcc4999 --- /dev/null +++ b/vendor/pyproject.nix/default.nix @@ -0,0 +1,22 @@ +{ lib }: +let + inherit (builtins) mapAttrs; + inherit (lib) fix; +in + +fix (self: mapAttrs (_: path: import path ({ inherit lib; } // self)) { + pip = ./pip.nix; + pypa = ./pypa.nix; + project = ./project.nix; + renderers = ./renderers.nix; + validators = ./validators.nix; + poetry = ./poetry.nix; + + pep427 = ./pep427.nix; + pep440 = ./pep440.nix; + pep508 = ./pep508.nix; + pep518 = ./pep518.nix; + pep599 = ./pep599.nix; + pep600 = ./pep600.nix; + pep621 = ./pep621.nix; +}) diff --git a/vendor/pyproject.nix/pep427.nix b/vendor/pyproject.nix/pep427.nix new file mode 100644 index 0000000..14ff390 --- /dev/null +++ b/vendor/pyproject.nix/pep427.nix @@ -0,0 +1,49 @@ +_: + +let + inherit (builtins) match elemAt split filter isString; + matchFileName = match "([^-]+)-([^-]+)(-([[:digit:]][^-]*))?-([^-]+)-([^-]+)-(.+).whl"; + +in +{ + /* Check whether string is a wheel file or not. + + Type: isWheelFileName :: string -> bool + + Example: + # isWheelFileName "cryptography-41.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + true + */ + isWheelFileName = name: matchFileName name != null; + + /* Parse PEP-427 wheel file names. + + Type: parseFileName :: string -> AttrSet + + Example: + # parseFileName "cryptography-41.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + { + abiTag = "abi3"; + buildTag = null; + distribution = "cryptography"; + languageTag = "cp37"; + platformTags = [ "manylinux_2_17_aarch64" "manylinux2014_aarch64" ]; + version = "41.0.1"; + } + */ + parseFileName = + # The wheel filename is `{distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl`. + name: + let + m = matchFileName name; + mAt = elemAt m; + in + assert m != null; { + distribution = mAt 0; + version = mAt 1; + buildTag = mAt 3; + languageTag = mAt 4; + abiTag = mAt 5; + platformTags = filter isString (split "\\." (mAt 6)); + }; +} diff --git a/vendor/pyproject.nix/pep440.nix b/vendor/pyproject.nix/pep440.nix new file mode 100644 index 0000000..0b5f834 --- /dev/null +++ b/vendor/pyproject.nix/pep440.nix @@ -0,0 +1,261 @@ +{ lib, ... }: +let + inherit (builtins) split filter match length elemAt head tail foldl' fromJSON typeOf; + inherit (lib) fix isString toInt toLower sublist; + + filterNull = filter (x: x != null); + filterEmpty = filter (x: length x > 0); + filterEmptyStr = filter (s: s != ""); + + # A version of lib.toInt that supports leading zeroes + toIntRelease = s: + let + n = fromJSON (head (match "0?([[:digit:]]+)" s)); + in + assert typeOf n == "int"; n; + + # Return a list elem at index with a default value if it doesn't exist + optionalElem = list: idx: default: if length list >= idx + 1 then elemAt list idx else default; + + # We consider some words to be alternate spellings of other words and + # in those cases we want to normalize the spellings to our preferred + # spelling. + normalizedReleaseTypes = { + alpha = "a"; + beta = "b"; + c = "rc"; + pre = "rc"; + preview = "rc"; + rev = "post"; + r = "post"; + "-" = "post"; + }; + + # Parse a release (pre/post/whatever) attrset from split tokens + parseReleaseSuffix = patterns: tokens: + let + matches = map + (x: + let + type = toLower (elemAt x 0); + value = elemAt x 1; + in + { + type = normalizedReleaseTypes.${type} or type; + value = if value != "" then toInt value else 0; + }) + (filterNull (map (match "[0-9]*(${patterns})([0-9]*)") tokens)); + in + assert length matches <= 1; optionalElem matches 0 null; + + parsePre = parseReleaseSuffix "a|b|c|rc|alpha|beta|pre|preview"; + parsePost = parseReleaseSuffix "post|rev|r|\-"; + parseDev = parseReleaseSuffix "dev"; + parseLocal = parseReleaseSuffix "\\+"; + + # Compare the release fields from the parsed version + compareRelease = ra: rb: + let + x = head ra; + y = head rb; + in + if length ra == 0 || length rb == 0 then 0 else + ( + if x == "*" || y == "*" then 0 # Wildcards are always considered equal + else + ( + if x > y then 1 + else if x < y then -1 + else compareRelease (tail ra) (tail rb) + ) + ); + + # Normalized modifier to it's priority (in case we are comparing an alpha to a beta or similar) + modifierPriority = { + dev = -1; + a = 0; + b = 1; + rc = 2; + post = 3; + }; + + # Compare dev/pre/post/local release modifiers + compareVersionModifier = x: y: assert x != null && y != null; let + prioX = modifierPriority.${x.type}; + prioY = modifierPriority.${y.type}; + in + if prioX == prioY then + ( + if x.value == y.value then 0 + else if x.value > y.value then 1 + else -1 + ) + else if prioX > prioY then 1 + else 0; + +in +fix (self: { + + /* Parse a version according to PEP-440. + + Type: parseVersion :: string -> AttrSet + + Example: + # parseVersion "3.0.0rc1" + { + dev = null; + epoch = 0; + local = null; + post = null; + pre = { + type = "rc"; + value = 1; + }; + release = [ 3 0 0 ]; + } + */ + parseVersion = version: + let + tokens = filter isString (split "\\." version); + in + { + # Return epoch defaulting to 0 + epoch = toInt (optionalElem (map head (filterNull (map (match "[0-9]+!([0-9]+)") tokens))) 0 "0"); + release = map (t: (x: if x == "*" then x else toIntRelease x) (head t)) (filterEmpty (map (t: filterEmptyStr (match "([\\*0-9]*).*" t)) tokens)); + pre = parsePre tokens; + post = parsePost tokens; + dev = parseDev tokens; + local = parseLocal tokens; + }; + + /* Parse a version conditional. + + Type: parseVersionCond :: string -> AttrSet + + Example: + # parseVersionCond ">=3.0.0rc1" + { + op = ">="; + version = { + dev = null; + epoch = 0; + local = null; + post = null; + pre = { + type = "rc"; + value = 1; + }; + release = [ 3 0 0 ]; + }; + } + */ + parseVersionCond = cond: ( + let + m = match " *([=> AttrSet -> int + + Example: + # compareVersions (parseVersion "3.0.0") (parseVersion "3.0.0") + 0 + */ + compareVersions = a: b: foldl' (acc: comp: if acc != 0 then acc else comp) 0 [ + # mixing dev/pre/post like: + # 1.0b2.post345.dev456 + # 1.0b2.post345 + # is valid and we need to consider them all. + + # Compare release field + (compareRelease a.release b.release) + + # Compare pre release + ( + if a.pre != null && b.pre != null then compareVersionModifier a.pre b.pre + else if a.pre != null then -1 + else if b.pre != null then 1 + else 0 + ) + + # Compare dev release + ( + if a.dev != null && b.dev != null then compareVersionModifier a.dev b.dev + else if a.dev != null then -1 + else if b.dev != null then 1 + else 0 + ) + + # Compare post release + ( + if a.post != null && b.post != null then compareVersionModifier a.post b.post + else if a.post != null then 1 + else if b.post != null then -1 + else 0 + ) + + # Compare epoch + ( + if a.epoch == b.epoch then 0 + else if a.epoch > b.epoch then 1 + else -1 + ) + + # Compare local + ( + if a.local != null && b.local != null then compareVersionModifier a.local b.local + else if b.local != null then -1 + else 0 + ) + ]; + + /* Map comparison operators as strings to a comparator function. + + Attributes: + - [Compatible release clause](https://peps.python.org/pep-0440/#compatible-release): `~=` + - [Version matching clause](https://peps.python.org/pep-0440/#version-matching): `==` + - [Version exclusion clause](https://peps.python.org/pep-0440/#version-exclusion): `!=` + - [Inclusive ordered comparison clause](https://peps.python.org/pep-0440/#inclusive-ordered-comparison): `<=`, `>=` + - [Exclusive ordered comparison clause](https://peps.python.org/pep-0440/#exclusive-ordered-comparison): `<`, `>` + - [Arbitrary equality clause](https://peps.python.org/pep-0440/#arbitrary-equality): `===` + + Type: operators.${operator} :: AttrSet -> AttrSet -> bool + + Example: + # comparators."==" (parseVersion "3.0.0") (parseVersion "3.0.0") + true + */ + comparators = { + "~=" = a: b: ( + # Local version identifiers are NOT permitted in this version specifier. + assert a.local == null && b.local == null; + self.comparators.">=" a b && self.comparators."==" a (b // { + release = sublist 0 ((length b.release) - 1) b.release; + # If a pre-release, post-release or developmental release is named in a compatible release clause as V.N.suffix, then the suffix is ignored when determining the required prefix match. + pre = null; + post = null; + dev = null; + }) + ); + "==" = a: b: self.compareVersions a b == 0; + "!=" = a: b: self.compareVersions a b != 0; + "<=" = a: b: self.compareVersions a b <= 0; + ">=" = a: b: self.compareVersions a b >= 0; + "<" = a: b: self.compareVersions a b < 0; + ">" = a: b: self.compareVersions a b > 0; + "===" = throw "Arbitrary equality clause not supported"; + }; + +}) diff --git a/vendor/pyproject.nix/pep508.nix b/vendor/pyproject.nix/pep508.nix new file mode 100644 index 0000000..04b0447 --- /dev/null +++ b/vendor/pyproject.nix/pep508.nix @@ -0,0 +1,544 @@ +{ lib, pep440, pep599, pypa, ... }: + +let + inherit (builtins) match elemAt split foldl' substring stringLength typeOf fromJSON isString head mapAttrs elem length; + inherit (lib) stringToCharacters fix; + inherit (import ./util.nix { inherit lib; }) splitComma; + + re = { + operators = "([=>= condPrio.${r}; + + # Parse a value into an attrset of { type = "valueType"; value = ...; } + # Will parse any field name suffixed with "version" as a PEP-440 version, otherwise + # the value is passed through and the type is inferred with builtins.typeOf + parseValueVersionDynamic = name: value: ( + if match "^.+version" name != null && isString value then { + type = "version"; + value = pep440.parseVersion value; + } else { + type = typeOf value; + inherit value; + } + ); + + # Strip leading/trailing whitespace from string + stripStr = s: let t = match "[\t ]*(.*[^\t ])[\t ]*" s; in if t == null then "" else head t; + + # Remove groupings ( ) from expression + unparen = expr': + let + expr = stripStr expr'; + m = match "\\((.+)\\)" expr; + in + if m != null then elemAt m 0 else expr; + + isMarkerVariable = + let + markerFields = [ + "implementation_name" + "implementation_version" + "os_name" + "platform_machine" + "platform_python_implementation" + "platform_release" + "platform_system" + "platform_version" + "python_full_version" + "python_version" + "sys_platform" + "extra" + ]; + in + s: elem s markerFields; + + unpackValue = value: + let + # If the value is a single ticked string we can't pass it plainly to toJSON. + # Normalise to a double quoted. + singleTicked = match "^'(.+)'$" value; # TODO: Account for escaped ' in input (unescape) + in + if isMarkerVariable value then value + else fromJSON (if singleTicked != null then "\"" + head singleTicked + "\"" else value); + + # Comparators for simple equality + # For versions see pep440.comparators + comparators = { + "==" = a: b: a == b; + "!=" = a: b: a != b; + "<=" = a: b: a <= b; + ">=" = a: b: a >= b; + "<" = a: b: a < b; + ">" = a: b: a > b; + "===" = a: b: a == b; + }; + + # Special case comparators for the `extra` environment field + extraComparators = { + # Check for member in list if list, otherwise simply compare. + "==" = extras: extra: if typeOf extras == "list" then elem extra extras else extras == extra; + "!=" = extras: extra: if typeOf extras == "list" then !(elem extra extras) else extras != extra; + }; + + boolOps = { + "and" = x: y: x && y; + "or" = x: y: x || y; + "in" = x: y: lib.strings.hasInfix x y; + }; + + isPrimitiveType = + let + primitives = [ + "int" + "float" + "string" + "bool" + ]; + in + type: elem type primitives; + +in +fix (self: +{ + + /* Parse PEP 508 markers into an AST. + + Type: parseMarkers :: string -> AttrSet + + Example: + # parseMarkers "(os_name=='a' or os_name=='b') and os_name=='c'" + { + lhs = { + lhs = { + lhs = { + type = "variable"; + value = "os_name"; + }; + op = "=="; + rhs = { + type = "string"; + value = "a"; + }; + type = "compare"; + }; + op = "or"; + rhs = { + lhs = { + type = "variable"; + value = "os_name"; + }; + op = "=="; + rhs = { + type = "string"; + value = "b"; + }; + type = "compare"; + }; + type = "boolOp"; + }; + op = "and"; + rhs = { + lhs = { + type = "variable"; + value = "os_name"; + }; + op = "=="; + rhs = { + type = "string"; + value = "c"; + }; + type = "compare"; + }; + type = "boolOp"; + } + */ + parseMarkers = input: + let + # Find the positions of lhs/op/rhs in the input string + pos = foldl' + (acc: c: + let + # # Look ahead to find the operator (either "and", "not" or "or"). + cond = + if self.openP > 0 || acc.inString then "" + else if substring acc.pos 5 input == " and " then "and" + else if substring acc.pos 4 input == " or " then "or" + else if substring acc.pos 4 input == " in " then "in" + else if substring acc.pos 8 input == " not in " then "not in" + else if substring acc.pos 5 input == " not " then "not" + else ""; + + # When we've reached the operator we know the start/end positions of lhs/op/rhs + rhsOffset = + if cond != "" && condGt cond acc.cond then + ( + if (cond == "and" || cond == "not") then 5 + else if (cond == "or" || cond == "in") then 4 + else if cond == "not in" then 8 + else throw "Unknown cond: ${cond}" + ) else -1; + + self = { + # If we are inside a string don't track the opening and closing of parens + openP = if acc.inString then acc.openP else + ( + if c == "(" then acc.openP + 1 + else if c == ")" then acc.openP - 1 + else acc.openP + ); + + # Check opening and closing of strings + inString = + if acc.inString && c == "'" then true + else if !acc.inString && c == "'" then false + else acc.inString; + + pos = acc.pos + 1; + + cond = if cond != "" then cond else acc.cond; + + lhs = if (rhsOffset != -1) then acc.pos else acc.lhs; + rhs = if (rhsOffset != -1) then (acc.pos + rhsOffset) else acc.rhs; + }; + + in + self) + { + openP = 0; # Number of open parens + inString = false; # If the parser is inside a string + pos = 0; # Parser position + done = false; + + # Keep track of last logical condition to do precedence ordering + cond = ""; + + # Stop positions for each value + lhs = -1; + rhs = -1; + + } + (stringToCharacters input); + + in + if pos.lhs == -1 then # No right hand value to extract + ( + let + m = split re.operators (unparen input); + mLength = length m; + mAt = elemAt m; + lhs = stripStr (mAt 0); + in + if (mLength > 1) then assert mLength == 3; { + type = "compare"; + lhs = + if isMarkerVariable lhs then { + type = "variable"; + value = lhs; + } else unpackValue lhs; + op = elemAt (mAt 1) 0; + rhs = parseValueVersionDynamic lhs (unpackValue (stripStr (mAt 2))); + } else if isMarkerVariable input then { + type = "variable"; + value = input; + } else rec { + value = unpackValue input; + type = typeOf value; + } + ) else { + type = "boolOp"; + lhs = self.parseMarkers (unparen (substring 0 pos.lhs input)); + op = substring (pos.lhs + 1) (pos.rhs - pos.lhs - 2) input; + rhs = self.parseMarkers (unparen (substring pos.rhs (stringLength input) input)); + }; + + /* Parse a PEP-508 dependency string. + + Type: parseString :: string -> AttrSet + + Example: + # parseString "cachecontrol[filecache]>=0.13.0" + { + conditions = [ + { + op = ">="; + version = { + dev = null; + epoch = 0; + local = null; + post = null; + pre = null; + release = [ 0 13 0 ]; + }; + } + ]; + markers = null; + name = "cachecontrol"; + extras = [ "filecache" ]; + url = null; + } + */ + parseString = input: + let + # Split the input into it's distinct parts: The package segment, URL and environment markers + tokens = + let + # Input has both @ and ; separators (both URL and markers) + # "name [fred,bar] @ http://foo.com ; python_version=='2.7'" + m1 = match "^(.+)@(.+);(.+)$" input; + + # Input has ; separator (markers) + # "name [fred,bar] ; python_version=='2.7'" + m2 = match "^(.+);(.+)$" input; + + # Input has @ separator (URL) + # "name [fred,bar] @ http://foo.com" + m3 = match "^(.+)@(.+)$" input; + + in + if m1 != null then { + packageSegment = elemAt m1 0; + url = stripStr (elemAt m1 1); + markerSegment = elemAt m1 2; + } + else if m2 != null then { + packageSegment = elemAt m2 0; + url = null; + markerSegment = elemAt m2 1; + } + else if m3 != null then { + packageSegment = elemAt m3 0; + url = stripStr (elemAt m3 1); + markerSegment = null; + } + else + ( + if match ".+\/.+" input != null then + # Input is a bare URL + { + packageSegment = null; + url = input; + markerSegment = null; + } else + # Input is a package name + { + packageSegment = input; + url = null; + markerSegment = null; + } + ); + + # Extract metadata from the package segment + package = + let + # Package has either both extras and version constraints or just extras + # "name [fred,bar]>=3.10" + # "name [fred,bar]" + m1 = match "(.+)\\[(.*)](.*)" tokens.packageSegment; + + # Package has either version constraints or is bare + # "name>=3.2" + # "name" + m2 = match "([a-zA-Z0-9_\\.-]+)(.*)" tokens.packageSegment; + + # The version conditions as a list of strings + conditions = map pep440.parseVersionCond (splitComma (if m1 != null then elemAt m1 2 else elemAt m2 1)); + + # Extras as a list of strings + # + # Based on PEP-508 alone it's not clear whether extras should be normalized or not. + # From discussion in https://github.com/pypa/packaging-problems/issues/230 + # missing normalization seems like an oversight. + extras = if m1 != null then map pypa.normalizePackageName (splitComma (elemAt m1 1)) else [ ]; + + in + if tokens.packageSegment == null then { + name = null; + conditions = [ ]; + extras = [ ]; + } else + # Assert that either regex matched + assert m1 != null || m2 != null; { + # Based on PEP-508 alone it's not clear whether names should be normalized or not. + # From discussion in https://github.com/pypa/packaging-problems/issues/230 + # this seems like an oversight and we _should_ actually canonicalize names at parse time. + name = pypa.normalizePackageName (stripStr (if m1 != null then elemAt m1 0 else elemAt m2 0)); + inherit extras conditions; + }; + + in + { + inherit (package) name conditions extras; + inherit (tokens) url; + markers = if tokens.markerSegment == null then null else self.parseMarkers tokens.markerSegment; + }; + + /* Create an attrset of platform variables. + As described in https://peps.python.org/pep-0508/#environment-markers. + + Type: mkEnviron :: derivation -> AttrSet + + Example: + # mkEnviron pkgs.python3 + { + implementation_name = { + type = "string"; + value = "cpython"; + }; + implementation_version = { + type = "version"; + value = { + dev = null; + epoch = 0; + local = null; + post = null; + pre = null; + release = [ 3 10 12 ]; + }; + }; + os_name = { + type = "string"; + value = "posix"; + }; + platform_machine = { + type = "string"; + value = "x86_64"; + }; + platform_python_implementation = { + type = "string"; + value = "CPython"; + }; + # platform_release maps to platform.release() which returns + # the running kernel version on Linux. + # Because this field is not reproducible it's left empty. + platform_release = { + type = "string"; + value = ""; + }; + platform_system = { + type = "string"; + value = "Linux"; + }; + # platform_version maps to platform.version() which also returns + # the running kernel version on Linux. + # Because this field is not reproducible it's left empty. + platform_version = { + type = "version"; + value = { + dev = null; + epoch = 0; + local = null; + post = null; + pre = null; + release = [ ]; + }; + }; + python_full_version = { + type = "version"; + value = { + dev = null; + epoch = 0; + local = null; + post = null; + pre = null; + release = [ 3 10 12 ]; + }; + }; + python_version = { + type = "version"; + value = { + dev = null; + epoch = 0; + local = null; + post = null; + pre = null; + release = [ 3 10 ]; + }; + }; + sys_platform = { + type = "string"; + value = "linux"; + }; + } + */ + mkEnviron = python: + let + inherit (python) stdenv; + targetMachine = pep599.manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name} or null; + in + mapAttrs + parseValueVersionDynamic + { + os_name = + if python.pname == "jython" then "java" + else "posix"; + sys_platform = + if stdenv.isLinux then "linux" + else if stdenv.isDarwin then "darwin" + else throw "Unsupported platform"; + platform_machine = targetMachine; + platform_python_implementation = + let + impl = python.passthru.implementation; + in + if impl == "cpython" then "CPython" + else if impl == "pypy" then "PyPy" + else throw "Unsupported implementation ${impl}"; + platform_release = ""; # Field not reproducible + platform_system = + if stdenv.isLinux then "Linux" + else if stdenv.isDarwin then "Darwin" + else throw "Unsupported platform"; + platform_version = ""; # Field not reproducible + python_version = python.passthru.pythonVersion; + python_full_version = python.version; + implementation_name = python.passthru.implementation; + implementation_version = python.version; + }; + + /* Evaluate an environment as returned by `mkEnviron` against markers as returend by `parseMarkers`. + + Type: evalMarkers :: AttrSet -> AttrSet -> bool + + Example: + # evalMarkers (mkEnviron pkgs.python3) (parseMarkers "python_version < \"3.11\"") + true + */ + evalMarkers = environ: value: ( + let + x = self.evalMarkers environ value.lhs; + y = self.evalMarkers environ value.rhs; + in + if value.type == "compare" then + ( + ( + # Version comparison + if value.lhs.type == "version" || value.rhs.type == "version" then pep440.comparators.${value.op} + # `Extra` environment marker comparison requires special casing because it's equality checks can + # == can be considered a `"key" in set` comparison when multiple extras are activated for a dependency. + # If we didn't treat it this way the check would become quadratic as `evalMarkers` only could check one extra at a time. + else if value.lhs.type == "variable" || value.lhs.value == "extra" then extraComparators.${value.op} + # Simple equality + else comparators.${value.op} + ) x + y + ) + else if value.type == "boolOp" then boolOps.${value.op} x y + else if value.type == "variable" then (self.evalMarkers environ environ.${value.value}) + else if value.type == "version" || value.type == "extra" then value.value + else if isPrimitiveType value.type then value.value + else throw "Unknown type '${value.type}'" + ); + +}) diff --git a/vendor/pyproject.nix/pep518.nix b/vendor/pyproject.nix/pep518.nix new file mode 100644 index 0000000..47198b8 --- /dev/null +++ b/vendor/pyproject.nix/pep518.nix @@ -0,0 +1,12 @@ +{ pep508, ... }: + +{ + /* Parse PEP-518 `build-system.requires` from pyproject.toml. + Type: readPyproject :: AttrSet -> list + + Example: + # parseBuildSystems (lib.importTOML ./pyproject.toml) + [ ] # List of parsed PEP-508 strings as returned by `lib.pep508.parseString`. + */ + parseBuildSystems = pyproject: map pep508.parseString (pyproject.build-system.requires or [ ]); +} diff --git a/vendor/pyproject.nix/pep599.nix b/vendor/pyproject.nix/pep599.nix new file mode 100644 index 0000000..f17fc7c --- /dev/null +++ b/vendor/pyproject.nix/pep599.nix @@ -0,0 +1,20 @@ +_: + +{ + /* Map Nixpkgs CPU values to target machines known to be supported for manylinux* wheels (a.k.a. `uname -m`), + in nixpkgs found under the attribute `stdenv.targetPlatform.parsed.cpu.name` + + Example: + # legacyAliases.powerpc64 + "ppc64" + */ + manyLinuxTargetMachines = { + x86_64 = "x86_64"; + i686 = "i686"; + aarch64 = "aarch64"; + armv7l = "armv7l"; + powerpc64 = "ppc64"; + powerpc64le = "ppc64le"; + s390x = "s390x"; + }; +} diff --git a/vendor/pyproject.nix/pep600.nix b/vendor/pyproject.nix/pep600.nix new file mode 100644 index 0000000..6a8d506 --- /dev/null +++ b/vendor/pyproject.nix/pep600.nix @@ -0,0 +1,58 @@ +{ lib, pep599, ... }: +let + inherit (builtins) match elemAt compareVersions splitVersion; + inherit (lib) fix; + +in +fix (self: { + /* Map legacy (pre PEP-600) platform tags to PEP-600 compliant ones. + + https://peps.python.org/pep-0600/#legacy-manylinux-tags + + Type: legacyAliases.${tag} :: AttrSet -> string + + Example: + # legacyAliases."manylinux1_x86_64" or "manylinux1_x86_64" + "manylinux_2_5_x86_64" + */ + legacyAliases = { + manylinux1_x86_64 = "manylinux_2_5_x86_64"; + manylinux1_i686 = "manylinux_2_5_i686"; + manylinux2010_x86_64 = "manylinux_2_12_x86_64"; + manylinux2010_i686 = "manylinux_2_12_i686"; + manylinux2014_x86_64 = "manylinux_2_17_x86_64"; + manylinux2014_i686 = "manylinux_2_17_i686"; + manylinux2014_aarch64 = "manylinux_2_17_aarch64"; + manylinux2014_armv7l = "manylinux_2_17_armv7l"; + manylinux2014_ppc64 = "manylinux_2_17_ppc64"; + manylinux2014_ppc64le = "manylinux_2_17_ppc64le"; + manylinux2014_s390x = "manylinux_2_17_s390x"; + }; + + /* Check if a manylinux tag is compatible with a given stdenv. + + Type: manyLinuxTagCompatible :: AttrSet -> string -> bool + + Example: + # manyLinuxTagCompatible pkgs.stdenv "manylinux_2_5_x86_64" + true + */ + manyLinuxTagCompatible = stdenv: tag: + let + tag' = self.legacyAliases.${tag} or tag; + m = match "manylinux_([0-9]+)_([0-9]+)_(.*)" tag'; + mAt = elemAt m; + tagMajor = mAt 0; + tagMinor = mAt 1; + tagArch = mAt 2; + sysVersion' = elemAt (splitVersion stdenv.cc.libc.version); + sysMajor = sysVersion' 0; + sysMinor = sysVersion' 1; + in + if m == null then throw "'${tag'}' is not a valid manylinux tag." + else if stdenv.cc.libc.pname != "glibc" then false + else if compareVersions "${sysMajor}.${sysMinor}" "${tagMajor}.${tagMinor}" < 0 then false + else if pep599.manyLinuxTargetMachines.${tagArch} != stdenv.targetPlatform.parsed.cpu.name then false + else true; + +}) diff --git a/vendor/pyproject.nix/pep621.nix b/vendor/pyproject.nix/pep621.nix new file mode 100644 index 0000000..13de814 --- /dev/null +++ b/vendor/pyproject.nix/pep621.nix @@ -0,0 +1,147 @@ +{ lib, pep440, pep508, pep518, ... }: +let + inherit (builtins) mapAttrs foldl' split filter elem; + inherit (lib) isString filterAttrs fix; + + splitAttrPath = path: filter isString (split "\\." path); + getAttrPath = path: lib.getAttrFromPath (splitAttrPath path); + +in +fix (self: { + /* Parse dependencies from pyproject.toml. + + Type: parseDependencies :: AttrSet -> AttrSet + + Example: + # parseDependencies { + # + # pyproject = (lib.importTOML ./pyproject.toml); + # # Don't just look at `project.optional-dependencies` for groups, also look at these: + # extrasAttrPaths = [ "tool.pdm.dev-dependencies" ]; + # } + { + dependencies = [ ]; # List of parsed PEP-508 strings (lib.pep508.parseString) + extras = { + dev = [ ]; # List of parsed PEP-508 strings (lib.pep508.parseString) + }; + build-systems = [ ]; # PEP-518 build-systems (List of parsed PEP-508 strings) + } + */ + parseDependencies = { pyproject, extrasAttrPaths ? [ ] }: + let + # Fold extras from all considered attributes into one set + extras' = foldl' (acc: attr: acc // getAttrPath attr pyproject) (pyproject.project.optional-dependencies or { }) extrasAttrPaths; + in + { + dependencies = map pep508.parseString (pyproject.project.dependencies or [ ]); + extras = mapAttrs (_: map pep508.parseString) extras'; + build-systems = pep518.parseBuildSystems pyproject; + }; + + /* Parse project.python-requires from pyproject.toml + + Type: parseRequiresPython :: AttrSet -> list + + Example: + # parseRequiresPython (lib.importTOML ./pyproject.toml) + [ ] # List of conditions as returned by `lib.pep440.parseVersionCond` + */ + parseRequiresPython = pyproject: map pep440.parseVersionCond (filter isString (split "," (pyproject.project.requires-python or ""))); + + /* Takes a dependency structure as returned by `lib.pep621.parseDependencies` and transforms it into + a structure with it's package names. + + Type: getDependenciesNames :: AttrSet -> AttrSet + + Example: + # getDependenciesNames (pep621.parseDependencies { pyproject = (lib.importTOML ./pyproject.toml); }) + { + dependencies = [ "requests" ]; + extras = { + dev = [ "pytest" ]; + }; + build-systems = [ "poetry-core" ]; + } + */ + getDependenciesNames = + let + getNames = map (dep: dep.name); + in + dependencies: { + dependencies = getNames dependencies.dependencies; + extras = mapAttrs (_: getNames) dependencies.extras; + build-systems = getNames dependencies.build-systems; + }; + /* Filter dependencies not relevant for this environment. + + Type: filterDependenciesByEnviron :: AttrSet -> AttrSet -> AttrSet + + Example: + # filterDependenciesByEnviron (lib.pep508.mkEnviron pkgs.python3) (lib.pep621.parseDependencies (lib.importTOML ./pyproject.toml)) + { } # Structure omitted in docs + */ + filterDependenciesByEnviron = + # Environ as created by `lib.pep508.mkEnviron`. + environ: + # Extras as a list of strings + extras: + # Dependencies as parsed by `lib.pep621.parseDependencies`. + dependencies: + ( + let + filterList = environ: filter (dep: dep.markers == null || pep508.evalMarkers + (environ // { + extra = { + type = "extra"; + value = extras; + }; + }) + dep.markers); + in + { + dependencies = filterList environ dependencies.dependencies; + extras = mapAttrs (_: filterList environ) dependencies.extras; + build-systems = filterList environ dependencies.build-systems; + } + ); + + /* Filter dependencies by their extras groups. + + Type: filterDependenciesByExtras :: list[string] -> AttrSet -> AttrSet + + Example: + # filterDependenciesByExtras [ "dev" ] (lib.pep621.parseDependencies (lib.importTOML ./pyproject.toml)) + { } # Structure omitted in docs + */ + filterDependenciesByExtras = + # Extras groups as a list of strings. + extras: + # Dependencies as parsed by `lib.pep621.parseDependencies`. + dependencies: + dependencies // { + extras = filterAttrs (group: _: elem group extras) dependencies.extras; + }; + + /* Aggregate of `filterDependencies` & `filterDependenciesByExtras` + + Type: filterDependencies :: AttrSet -> AttrSet + + Example: + # filterDependencies { + # dependencies = lib.pep621.parseDependencies (lib.importTOML ./pyproject.toml); + # environ = lib.pep508.mkEnviron pkgs.python; + # extras = [ "dev" ]; + # } + { } # Structure omitted in docs + */ + filterDependencies = + { + # Dependencies as parsed by `lib.pep621.parseDependencies` + dependencies + , # Environ as created by `lib.pep508.mkEnviron` + environ + , # Extras as a list of strings + extras ? [ ] + , + }: self.filterDependenciesByEnviron environ extras (self.filterDependenciesByExtras extras dependencies); +}) diff --git a/vendor/pyproject.nix/pip.nix b/vendor/pyproject.nix/pip.nix new file mode 100644 index 0000000..8f958ca --- /dev/null +++ b/vendor/pyproject.nix/pip.nix @@ -0,0 +1,105 @@ +{ lib +, pep508 +, ... +}: +let + inherit (builtins) match head tail typeOf split filter foldl' readFile dirOf hasContext unsafeDiscardStringContext; + + stripStr = s: + let + t = match "[\t ]*(.*[^\t ])[\t ]*" s; + in + if t == null + then "" + else head t; + + uncomment = l: head (match " *([^#]*).*" l); + +in +lib.fix (self: { + + /* Parse dependencies from requirements.txt + + Type: parseRequirementsTxt :: AttrSet -> list + + Example: + # parseRequirements ./requirements.txt + [ { flags = []; requirement = {}; # Returned by pep508.parseString } ] + */ + + parseRequirementsTxt = + # The contents of or path to requirements.txt + requirements: + let + # Paths are either paths or strings with context. + # Preferably we'd just use paths but because of + # + # $ ./. + requirements + # "a string that refers to a store path cannot be appended to a path" + # + # We also need to support stringly paths... + isPath = typeOf requirements == "path" || hasContext requirements; + path' = + if typeOf requirements == "path" then requirements else + /. + unsafeDiscardStringContext requirements; + root = dirOf path'; + + # Requirements without comments and no empty strings + requirements' = if isPath then readFile path' else requirements; + lines' = filter (l: l != "") (map uncomment (filter (l: typeOf l == "string") (split "\n" requirements'))); + # Fold line continuations + inherit ((foldl' + ( + acc: l': + let + m = match "(.+) *\\\\" l'; + continue = m != null; + l = stripStr ( + if continue + then (head m) + else l' + ); + in + if continue + then { + line = acc.line ++ [ l ]; + inherit (acc) lines; + } + else { + line = [ ]; + lines = acc.lines ++ [ (acc.line ++ [ l ]) ]; + } + ) + { + lines = [ ]; + line = [ ]; + } + lines')) lines; + + in + foldl' + (acc: l: + let + m = match "-(c|r) (.+)" (head l); + in + acc ++ ( + # Common case, parse string + if m == null + then [{ + requirement = pep508.parseString (head l); + flags = tail l; + }] + + # Don't support constraint files + else if (head m) == "c" then throw "Unsupported flag: -c" + + # Recursive requirements.txt + else + (self.parseRequirementsTxt ( + if root == null then throw "When importing recursive requirements.txt requirements needs to be passed as a path" + else root + "/${head (tail m)}" + )) + )) + [ ] + lines; +}) diff --git a/vendor/pyproject.nix/poetry.nix b/vendor/pyproject.nix/poetry.nix new file mode 100644 index 0000000..290bf2e --- /dev/null +++ b/vendor/pyproject.nix/poetry.nix @@ -0,0 +1,249 @@ +{ lib +, pep440 +, pep508 +, pep518 +, ... +}: +let + inherit (builtins) match elemAt foldl' typeOf attrNames head tail mapAttrs; + inherit (lib) optionalAttrs flatten; + inherit (import ./util.nix { inherit lib; }) splitComma; + + # Translate author from a string like "Name " to a structured set as defined by PEP-621. + translateAuthor = a: + let + mAt = elemAt (match "^(.+) <(.+)>$" a); + in + { name = mAt 0; email = mAt 1; }; + + # Normalize dependecy from poetry dependencies table from (string || set) -> set + normalizeDep = name: dep: ( + let + type = typeOf dep; + in + if type == "string" then { + inherit name; + version = dep; + } + else if type == "set" then dep // { inherit name; } + else throw "Unexpected type: ${type}" + ); + + # Rewrite the right hand side version for caret comparisons according to the rules laid out in + # https://python-poetry.org/docs/dependency-specification/#caret-requirements + rewriteCaretRhs = release: + let + state = foldl' + (state: v: + let + nonzero = state.nonzero || v != 0; + in + state // { + release = state.release ++ [ + ( + if nonzero && !state.nonzero then (v + 1) + else if nonzero then 0 + else v + ) + ]; + inherit nonzero; + }) + { + release = [ ]; + nonzero = false; + } + release; + in + if !state.nonzero + then ([ (head state.release + 1) ] ++ tail state.release) + else state.release; + + # Poetry dependency tables are of mixed types: + # [tool.poetry.dependencies] + # python = "^3.8" + # cachecontrol = { version = "^0.13.0", extras = ["filecache"] } + # foo = [ + # {version = "<=1.9", python = ">=3.6,<3.8"}, + # {version = "^2.0", python = ">=3.8"} + # ] + # + # These are all valid. Normalize the input to a list of: + # [ + # { name = "python"; version = "^3.8"; } + # { name = "cachecontrol"; version = "^0.13.0"; extras = ["filecache"]; } + # { name = "foo"; version = "<=1.9"; python = ">=3.6,<3.8"; } + # { name = "foo"; version = "^2.0"; python = ">=3.8"; } + # ] + normalizeDependendenciesToList = deps: foldl' + (acc: name: acc ++ ( + let + dep = deps.${name}; + in + if typeOf dep == "list" then map (normalizeDep name) dep + else [ (normalizeDep name dep) ] + )) [ ] + (attrNames deps); + + # Supports additional non-standard operators `^` and `~` used by Poetry. + # Other operators are passed through to pep440. + # Because some expressions desugar to multiple expressions parseVersionCond returns a list. + parseVersionCond' = cond: ( + let + m = match "^([~[:digit:]^])(.+)$" cond; + mAt = elemAt m; + c = mAt 0; + rest = mAt 1; + # Pad version before parsing as it's _much_ easier to reason about + # once they're the same length + version = pep440.parseVersion (lib.versions.pad 3 rest); + in + if m == null then [ (pep440.parseVersionCond cond) ] + # Desugar ~ into >= && < + else if c == "~" then [ + { + cond = ">="; + inherit version; + } + { + cond = "<"; + version = version // { + release = [ (head version.release + 1) ] ++ tail version.release; + }; + } + ] + # Desugar ^ into >= && < + else if c == "^" then [ + { + cond = ">="; + inherit version; + } + { + cond = "<"; + version = version // { + release = rewriteCaretRhs version.release; + }; + } + ] + # Versions without operators are exact matches, add operator according to PEP-440 + else [{ + cond = "=="; + inherit version; + }] + ); + + # Normalized version of parseVersionCond' + parseVersionConds = s: flatten (map parseVersionCond' (splitComma s)); + + dummyMarker = { + type = "bool"; + value = true; + }; + + # Analogous to pep508.parseString + parseDependency = dep: + let + # Poetry has Python as a separate field in the structured dependency object. + # This is non-standard. Rewrite these expressions as a nested set of logical ANDs that + # looks like regular parsed markers as if they were standard PEP-508, just written in a bit of a funky + # nested way that no human would do. + markers = foldl' + (rhs: pyCond: { + type = "boolOp"; + op = "and"; + lhs = { + type = "compare"; + inherit (pyCond) op; + lhs = { + type = "variable"; + value = "python_version"; + }; + rhs = { + type = "version"; + value = pyCond.version; + }; + }; + inherit rhs; + }) + ( + # Encode no markers as a marker that always evaluates to true to simplify fold logi above. + if dep ? markers then pep508.parseMarkers dep.markers else dummyMarker + ) + (if dep ? python then parseVersionConds dep.python else [ ]); + + in + { + inherit (dep) name; + conditions = parseVersionConds dep.version; + extras = dep.extras or [ ]; + url = dep.url or null; + markers = if markers == dummyMarker then null else markers; + }; + +in +{ + /* + Translate a Pyproject.toml from Poetry to PEP-621 project metadata. + This function transposes a PEP-621 project table on top of an existing Pyproject.toml populated with data from `tool.poetry`. + Notably does not translate dependencies/optional-dependencies. + + For parsing dependencies from Poetry see `lib.poetry.parseDependencies`. + + Type: translatePoetryProject :: AttrSet -> lambda + + Example: + # translatePoetryProject (lib.importTOML ./pyproject.toml) + { } # TOML contents, structure omitted. See PEP-621 for more information on data members. + */ + translatePoetryProject = pyproject: assert !(pyproject ? project); let + inherit (pyproject.tool) poetry; + in + pyproject // { + project = { + inherit (poetry) name version description; + authors = map translateAuthor poetry.authors; + urls = optionalAttrs (poetry ? homepage) + { + Homepage = poetry.homepage; + } // optionalAttrs (poetry ? repository) { + Repository = poetry.repository; + } // optionalAttrs (poetry ? documentation) { + Documentation = poetry.documentation; + }; + } // optionalAttrs (poetry ? license) { + license.text = poetry.license; + } // optionalAttrs (poetry ? maintainers) { + maintainers = map translateAuthor poetry.maintainers; + } // optionalAttrs (poetry ? readme) { + inherit (poetry) readme; + } // optionalAttrs (poetry ? keywords) { + inherit (poetry) keywords; + } // optionalAttrs (poetry ? classifiers) { + inherit (poetry) classifiers; + }; + }; + + /* Parse dependencies from pyproject.toml (Poetry edition). + This function is analogous to `lib.pep621.parseDependencies`. + + Type: parseDependencies :: AttrSet -> AttrSet + + Example: + # parseDependencies { + # + # pyproject = (lib.importTOML ./pyproject.toml); + # } + { + dependencies = [ ]; # List of parsed PEP-508 strings (lib.pep508.parseString) + extras = { + dev = [ ]; # List of parsed PEP-508 strings (lib.pep508.parseString) + }; + build-systems = [ ]; # PEP-518 build-systems (List of parsed PEP-508 strings) + } + */ + # # Analogous to + parseDependencies = pyproject: { + dependencies = map parseDependency (normalizeDependendenciesToList (pyproject.tool.poetry.dependencies or { })); + extras = mapAttrs (_: g: map parseDependency (normalizeDependendenciesToList g.dependencies)) pyproject.tool.poetry.group or { }; + build-systems = pep518.parseBuildSystems pyproject; + }; +} diff --git a/vendor/pyproject.nix/project.nix b/vendor/pyproject.nix/project.nix new file mode 100644 index 0000000..e29d1aa --- /dev/null +++ b/vendor/pyproject.nix/project.nix @@ -0,0 +1,82 @@ +{ pep518, pep621, poetry, pip, ... }: + +{ + /* Load dependencies from a pyproject.toml. + + Type: loadPyproject :: AttrSet -> AttrSet + + Example: + # loadPyproject { pyproject = lib.importTOML } + { + dependencies = { }; # Parsed dependency structure in the schema of `lib.pep621.parseDependencies` + build-systems = [ ]; # Returned by `lib.pep518.parseBuildSystems` + pyproject = { }; # The unmarshaled contents of pyproject.toml + } + */ + loadPyproject = + { + # The unmarshaled contents of pyproject.toml + pyproject + # Example: extrasAttrPaths = [ "tool.pdm.dev-dependencies" ]; + , extrasAttrPaths ? [ ] + }: { + dependencies = pep621.parseDependencies { inherit pyproject extrasAttrPaths; }; + build-systems = pep518.parseBuildSystems pyproject; + inherit pyproject; + }; + + /* Load dependencies from a Poetry pyproject.toml. + + Type: loadPoetryPyproject :: AttrSet -> AttrSet + + Example: + # loadPoetryPyproject { pyproject = lib.importTOML } + { + dependencies = { }; # Parsed dependency structure in the schema of `lib.pep621.parseDependencies` + build-systems = [ ]; # Returned by `lib.pep518.parseBuildSystems` + pyproject = { }; # The unmarshaled contents of pyproject.toml + } + */ + loadPoetryPyproject = + { + # The unmarshaled contents of pyproject.toml + pyproject + }: + let + pyproject-pep621 = poetry.translatePoetryProject pyproject; + in + { + dependencies = poetry.parseDependencies pyproject; + build-systems = pep518.parseBuildSystems pyproject; + pyproject = pyproject-pep621; + pyproject-poetry = pyproject; + }; + + /* Load dependencies from a requirements.txt. + + Note that as requirements.txt is lacking important project metadata this is incompatible with some renderers. + + Type: loadRequirementsTxt :: AttrSet -> AttrSet + + Example: + # loadRequirementstxt { requirements = builtins.readFile ./requirements.txt; root = ./.; } + { + dependencies = { }; # Parsed dependency structure in the schema of `lib.pep621.parseDependencies` + build-systems = [ ]; # Returned by `lib.pep518.parseBuildSystems` + pyproject = null; # The unmarshaled contents of pyproject.toml + } + */ + loadRequirementsTxt = + { + # The contents of requirements.txt + requirements + }: { + dependencies = { + dependencies = map (x: x.requirement) (pip.parseRequirementsTxt requirements); + extras = { }; + build-systems = [ ]; + }; + build-systems = [ ]; + pyproject = null; + }; +} diff --git a/vendor/pyproject.nix/pypa.nix b/vendor/pyproject.nix/pypa.nix new file mode 100644 index 0000000..deaa6aa --- /dev/null +++ b/vendor/pyproject.nix/pypa.nix @@ -0,0 +1,22 @@ +{ lib, ... }: +let + inherit (builtins) concatStringsSep filter split; + inherit (lib) isString toLower; + +in +{ + /* Normalize package name as documented in https://packaging.python.org/en/latest/specifications/name-normalization/#normalization + + Type: normalizePackageName :: string -> string + + Example: + # readPyproject "Friendly-Bard" + "friendly-bard" + */ + normalizePackageName = + let + concatDash = concatStringsSep "-"; + splitSep = split "[-_\.]+"; + in + name: toLower (concatDash (filter isString (splitSep name))); +} diff --git a/vendor/pyproject.nix/renderers.nix b/vendor/pyproject.nix/renderers.nix new file mode 100644 index 0000000..8707786 --- /dev/null +++ b/vendor/pyproject.nix/renderers.nix @@ -0,0 +1,147 @@ +{ lib +, pep508 +, pep621 +, ... +}: +let + inherit (builtins) attrValues length attrNames head foldl'; + inherit (lib) optionalAttrs flatten mapAttrs' filterAttrs; + + # Group licenses by their SPDX IDs for easy lookup + licensesBySpdxId = mapAttrs' + (_: license: { + name = license.spdxId; + value = license; + }) + (filterAttrs (_: license: license ? spdxId) lib.licenses); + +in +{ + /* + Renders a project as an argument that can be passed to withPackages + + Evaluates PEP-508 environment markers to select correct dependencies for the platform but does not validate version constraints. + For validation see `lib.validators`. + + Type: withPackages :: AttrSet -> lambda + + Example: + # withPackages (lib.project.loadPyproject { ... }) + «lambda @ «string»:1:1» + */ + withPackages = + { + # Project metadata as returned by `lib.project.loadPyproject` + project + , # Python derivation + python + , # Python extras (optionals) to enable + extras ? [ ] + , + }: + let + filteredDeps = pep621.filterDependencies { + inherit (project) dependencies; + environ = pep508.mkEnviron python; + inherit extras; + }; + namedDeps = pep621.getDependenciesNames filteredDeps; + flatDeps = namedDeps.dependencies ++ flatten (attrValues namedDeps.extras) ++ namedDeps.build-systems; + in + ps: map (dep: ps.${dep}) flatDeps; + + /* + Renders a project as an argument that can be passed to buildPythonPackage/buildPythonApplication. + + Evaluates PEP-508 environment markers to select correct dependencies for the platform but does not validate version constraints. + For validation see `lib.validators`. + + Type: buildPythonPackage :: AttrSet -> AttrSet + + Example: + # buildPythonPackage { project = lib.project.loadPyproject ...; python = pkgs.python3; } + { pname = "blinker"; version = "1.3.3.7"; propagatedBuildInputs = [ ]; } + */ + buildPythonPackage = + { + # Project metadata as returned by `lib.project.loadPyproject` + project + , # Python derivation + python + , # Python extras (optionals) to enable + extras ? [ ] + , # Map a Python extras group name to a Nix attribute set. + # This is intended to be used with optionals such as test dependencies that you might + # want to add to checkInputs instead of propagatedBuildInputs + extrasAttrMappings ? { } + , # Which package format to pass to buildPythonPackage + # If the format is "wheel" PEP-518 build-systems are excluded from the build. + format ? "pyproject" + }: + let + filteredDeps = pep621.filterDependencies { + inherit (project) dependencies; + environ = pep508.mkEnviron python; + inherit extras; + }; + + namedDeps = pep621.getDependenciesNames filteredDeps; + + inherit (project) pyproject; + + meta = + let + project' = project.pyproject.project; + urls = project'.urls or { }; + in + # Optional changelog + optionalAttrs (urls ? changelog) + { + inherit (urls) changelog; + } // + # Optional description + optionalAttrs (project' ? description) { + inherit (project') description; + } // + # Optional license + optionalAttrs (project'.license ? text) ( + assert !(project'.license ? file); { + # From PEP-621: + # "The text key has a string value which is the license of the project whose meaning is that of the License field from the core metadata. + # These keys are mutually exclusive, so a tool MUST raise an error if the metadata specifies both keys." + # Hence the assert above. + license = licensesBySpdxId.${project'.license.text}; + } + ) // + # Only set mainProgram if we only have one script, otherwise it's ambigious which one is main + ( + let + scriptNames = attrNames project'.scripts; + in + optionalAttrs (project' ? scripts && length scriptNames == 1) { + mainProgram = head scriptNames; + } + ); + + in + foldl' + (attrs: group: + let + attr = extrasAttrMappings.${group} or "propagatedBuildInputs"; + in + attrs // { + ${attr} = attrs.${attr} or [ ] ++ map (dep: python.pkgs.${dep}) namedDeps.extras.${group}; + }) + ({ + propagatedBuildInputs = map (dep: python.pkgs.${dep}) namedDeps.dependencies; + inherit format meta; + } // optionalAttrs (format != "wheel") { + nativeBuildInputs = map (dep: python.pkgs.${dep}) namedDeps.build-systems; + } // optionalAttrs (pyproject.project ? name) { + pname = pyproject.project.name; + } + // optionalAttrs (pyproject.project ? version) { + inherit (pyproject.project) version; + }) + (attrNames namedDeps.extras); +} diff --git a/vendor/pyproject.nix/util.nix b/vendor/pyproject.nix/util.nix new file mode 100644 index 0000000..143d923 --- /dev/null +++ b/vendor/pyproject.nix/util.nix @@ -0,0 +1,11 @@ +# Small utilities for internal reuse, not exposed externally +{ lib }: +let + inherit (builtins) filter match split; + inherit (lib) isString; + + isEmptyStr = s: isString s && match " *" s == null; +in +{ + splitComma = s: if s == "" then [ ] else filter isEmptyStr (split " *, *" s); +} diff --git a/vendor/pyproject.nix/validators.nix b/vendor/pyproject.nix/validators.nix new file mode 100644 index 0000000..6e2e117 --- /dev/null +++ b/vendor/pyproject.nix/validators.nix @@ -0,0 +1,71 @@ +{ lib +, pep440 +, pep508 +, pep621 +, pypa +, ... +}: +let + inherit (builtins) attrValues foldl' filter; + inherit (lib) flatten; + +in +{ + /* + Validates the Python package set held by Python (`python.pkgs`) against the parsed project. + + Returns an attribute set where the name is the Python package derivation `pname` and the value is a list of the mismatching conditions. + + Type: validateVersionConstraints :: AttrSet -> AttrSet + + Example: + # validateVersionConstraints (lib.project.loadPyproject { ... }) + { + resolvelib = { + # conditions as returned by `lib.pep440.parseVersionCond` + conditions = [ { op = ">="; version = { dev = null; epoch = 0; local = null; post = null; pre = null; release = [ 1 0 1 ]; }; } ]; + # Version from Python package set + version = "0.5.5"; + }; + unearth = { + conditions = [ { op = ">="; version = { dev = null; epoch = 0; local = null; post = null; pre = null; release = [ 0 10 0 ]; }; } ]; + version = "0.9.1"; + }; + } + */ + validateVersionConstraints = + { + # Project metadata as returned by `lib.project.loadPyproject` + project + , # Python derivation + python + , # Python extras (optionals) to enable + extras ? [ ] + , + }: + let + filteredDeps = pep621.filterDependencies { + inherit (project) dependencies; + environ = pep508.mkEnviron python; + inherit extras; + }; + flatDeps = filteredDeps.dependencies ++ flatten (attrValues filteredDeps.extras) ++ filteredDeps.build-systems; + + in + foldl' + (acc: dep: + let + pname = pypa.normalizePackageName dep.name; + pversion = python.pkgs.${pname}.version; + version = pep440.parseVersion python.pkgs.${pname}.version; + incompatible = filter (cond: ! pep440.comparators.${cond.op} version cond.version) dep.conditions; + in + if incompatible == [ ] then acc else acc // { + ${pname} = { + version = pversion; + conditions = incompatible; + }; + }) + { } + flatDeps; +} From b7617a7c4cf22aa19ae81d433ce5a8c166ef7013 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Oct 2023 14:44:12 +1300 Subject: [PATCH 3/4] pep508: Reimplement using pyproject.nix --- default.nix | 24 +- editable.nix | 1 - mk-poetry-dep.nix | 15 +- pep508.nix | 250 ------------------ .../in-list/{test_sqlalchemy.py => inlist.py} | 0 tests/in-list/pyproject.toml | 2 +- 6 files changed, 29 insertions(+), 263 deletions(-) delete mode 100644 pep508.nix rename tests/in-list/{test_sqlalchemy.py => inlist.py} (100%) diff --git a/default.nix b/default.nix index c59023d..ab262ad 100644 --- a/default.nix +++ b/default.nix @@ -5,6 +5,8 @@ let inherit (poetryLib) isCompatible readTOML normalizePackageName normalizePackageSet; + pyproject-nix = import ./vendor/pyproject.nix { inherit lib; }; + # Map SPDX identifiers to license names spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses))); # Get license by id falling back to input string @@ -104,7 +106,8 @@ lib.makeScope pkgs.newScope (self: { }: assert editablePackageSources != { }; import ./editable.nix { - inherit pyProject python pkgs lib poetryLib editablePackageSources; + inherit pyProject python pkgs lib editablePackageSources; + inherit pyproject-nix; }; /* Returns a package containing scripts defined in tool.poetry.scripts. @@ -141,11 +144,6 @@ lib.makeScope pkgs.newScope (self: { , extras ? [ "*" ] }: let - /* The default list of poetry2nix override overlays */ - mkEvalPep508 = import ./pep508.nix { - inherit lib poetryLib; - inherit (python) stdenv; - }; getFunctorFn = fn: if builtins.typeOf fn == "set" then fn.__functor else fn; scripts = pyProject.tool.poetry.scripts or { }; @@ -170,12 +168,19 @@ lib.makeScope pkgs.newScope (self: { in lib.listToAttrs (lib.mapAttrsToList (n: v: { name = normalizePackageName n; value = v; }) lockfiles); - evalPep508 = mkEvalPep508 python; + pep508Env = pyproject-nix.pep508.mkEnviron python; # Filter packages by their PEP508 markers & pyproject interpreter version partitions = let - supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true && isCompatible (poetryLib.getPythonVersion python) pkgMeta.python-versions; + supportsPythonVersion = pkgMeta: + if pkgMeta ? marker then + ( + let + marker = pyproject-nix.pep508.parseMarkers pkgMeta.marker; + in + pyproject-nix.pep508.evalMarkers pep508Env marker + ) else true && isCompatible (poetryLib.getPythonVersion python) pkgMeta.python-versions; in lib.partition supportsPythonVersion poetryLock.package; compatible = partitions.right; @@ -242,7 +247,8 @@ lib.makeScope pkgs.newScope (self: { self: super: { mkPoetryDep = self.callPackage ./mk-poetry-dep.nix { - inherit lib python poetryLib evalPep508; + inherit lib python poetryLib pep508Env; + inherit pyproject-nix; }; __toPluginAble = toPluginAble self; diff --git a/editable.nix b/editable.nix index 13a07e4..de33966 100644 --- a/editable.nix +++ b/editable.nix @@ -1,6 +1,5 @@ { pkgs , lib -, poetryLib , pyProject , python , editablePackageSources diff --git a/mk-poetry-dep.nix b/mk-poetry-dep.nix index 947e5df..cbbb3f0 100644 --- a/mk-poetry-dep.nix +++ b/mk-poetry-dep.nix @@ -3,11 +3,13 @@ , python , buildPythonPackage , poetryLib -, evalPep508 +, pep508Env +, pyproject-nix }: { name , version , pos ? __curPos +, extras ? [ ] , files , source , dependencies ? { } @@ -141,7 +143,16 @@ pythonPackages.callPackage constraints = v.python or ""; pep508Markers = v.markers or ""; in - compat constraints && evalPep508 pep508Markers + compat constraints && (if pep508Markers == "" then true else + (pyproject-nix.pep508.evalMarkers + (pep508Env // { + extra = { + # All extras are always enabled + type = "extra"; + value = lib.attrNames extras; + }; + }) + (pyproject-nix.pep508.parseMarkers pep508Markers))) ) dependencies ); diff --git a/pep508.nix b/pep508.nix deleted file mode 100644 index 65c6a49..0000000 --- a/pep508.nix +++ /dev/null @@ -1,250 +0,0 @@ -{ lib, stdenv, poetryLib }: python: -let - inherit (poetryLib) ireplace; - - targetMachine = poetryLib.getTargetMachine stdenv; - - # Like builtins.substring but with stop being offset instead of length - substr = start: stop: s: builtins.substring start (stop - start) s; - - # Strip leading/trailing whitespace from string - stripStr = s: lib.elemAt (builtins.split "^ *" (lib.elemAt (builtins.split " *$" s) 0)) 2; - findSubExpressionsFun = acc: c: ( - if c == "(" then - ( - let - posNew = acc.pos + 1; - isOpen = acc.openP == 0; - startPos = if isOpen then posNew else acc.startPos; - in - acc // { - inherit startPos; - exprs = acc.exprs ++ [ (substr acc.exprPos (acc.pos - 1) acc.expr) ]; - pos = posNew; - openP = acc.openP + 1; - } - ) else if c == ")" then - ( - let - openP = acc.openP - 1; - exprs = findSubExpressions (substr acc.startPos acc.pos acc.expr); - in - acc // { - inherit openP; - pos = acc.pos + 1; - exprs = if openP == 0 then acc.exprs ++ [ exprs ] else acc.exprs; - exprPos = if openP == 0 then acc.pos + 1 else acc.exprPos; - } - ) else acc // { pos = acc.pos + 1; } - ); - - # Make a tree out of expression groups (parens) - findSubExpressions = expr': - let - expr = " " + expr'; - acc = builtins.foldl' - findSubExpressionsFun - { - exprs = [ ]; - inherit expr; - pos = 0; - openP = 0; - exprPos = 0; - startPos = 0; - } - (lib.stringToCharacters expr); - tailExpr = substr acc.exprPos acc.pos expr; - tailExprs = if tailExpr != "" then [ tailExpr ] else [ ]; - in - acc.exprs ++ tailExprs; - parseExpressions = exprs: - let - splitCond = s: builtins.map - (x: stripStr (if builtins.typeOf x == "list" then (builtins.elemAt x 0) else x)) - (builtins.split " (and|or) " (s + " ")); - mapfn = expr: ( - if (builtins.match "^ ?$" expr != null) then null # Filter empty - else if (builtins.elem expr [ "and" "or" ]) then { - type = "bool"; - value = expr; - } - else { - type = "expr"; - value = expr; - } - ); - parse = expr: builtins.filter (x: x != null) (builtins.map mapfn (splitCond expr)); - in - builtins.foldl' - ( - acc: v: acc ++ (if builtins.typeOf v == "string" then parse v else [ (parseExpressions v) ]) - ) [ ] - exprs; - - # Transform individual expressions to structured expressions - # This function also performs variable substitution, replacing environment markers with their explicit values - transformExpressions = exprs: - let - variables = { - os_name = - if python.pname == "jython" then "java" - else "posix"; - sys_platform = - if stdenv.isLinux then "linux" - else if stdenv.isDarwin then "darwin" - else throw "Unsupported platform"; - platform_machine = targetMachine; - platform_python_implementation = - let - impl = python.passthru.implementation; - in - if impl == "cpython" then "CPython" - else if impl == "pypy" then "PyPy" - else throw "Unsupported implementation ${impl}"; - platform_release = ""; # Field not reproducible - platform_system = - if stdenv.isLinux then "Linux" - else if stdenv.isDarwin then "Darwin" - else throw "Unsupported platform"; - platform_version = ""; # Field not reproducible - python_version = python.passthru.pythonVersion; - python_full_version = python.version; - implementation_name = python.implementation; - implementation_version = python.version; - # extra = ""; - }; - substituteVar = value: if builtins.hasAttr value variables then (builtins.toJSON variables."${value}") else value; - processVar = value: builtins.foldl' (acc: v: v acc) value [ - stripStr - substituteVar - ]; - in - if builtins.typeOf exprs == "set" then - ( - if exprs.type == "expr" then - ( - let - mVal = ''[a-zA-Z0-9\'"_\. \-]+''; - mOp = "in|[!=<>]+"; - e = stripStr exprs.value; - m' = builtins.match ''^(${mVal}) +(${mOp}) *(${mVal})$'' e; - m = builtins.map stripStr (if m' != null then m' else builtins.match ''^(${mVal}) +(${mOp}) *(${mVal})$'' e); - m0 = processVar (builtins.elemAt m 0); - m2 = processVar (builtins.elemAt m 2); - in - { - type = "expr"; - value = { - # HACK: We don't know extra at eval time, so we assume the expression is always true - op = if m0 == "extra" then "true" else builtins.elemAt m 1; - values = [ m0 m2 ]; - }; - } - ) else exprs - ) else builtins.map transformExpressions exprs; - - # Recursively eval all expressions - evalExpressions = exprs: - let - unmarshal = v: ( - # TODO: Handle single quoted values - if v == "True" then true - else if v == "False" then false - else builtins.fromJSON v - ); - op = { - "true" = _x: _y: true; - "<=" = x: y: op.">=" y x; - "<" = x: y: lib.versionOlder (unmarshal x) (unmarshal y); - "!=" = x: y: x != y; - "==" = x: y: x == y; - ">=" = x: y: lib.versionAtLeast (unmarshal x) (unmarshal y); - ">" = x: y: op."<" y x; - "~=" = v: c: - let - parts = builtins.splitVersion c; - pruned = lib.take ((builtins.length parts) - 1) parts; - upper = builtins.toString ( - (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1 - ); - upperConstraint = builtins.concatStringsSep "." (ireplace (builtins.length pruned - 1) upper pruned); - in - op.">=" v c && op."<" v upperConstraint; - "===" = x: y: x == y; - "in" = x: y: - let - values = builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " (unmarshal y)); - in - builtins.elem (unmarshal x) values; - }; - in - if builtins.typeOf exprs == "set" then - ( - if exprs.type == "expr" then - ( - let - expr = exprs; - result = op."${expr.value.op}" (builtins.elemAt expr.value.values 0) (builtins.elemAt expr.value.values 1); - in - { - type = "value"; - value = result; - } - ) else exprs - ) else builtins.map evalExpressions exprs; - - # Now that we have performed an eval all that's left to do is to concat the graph into a single bool - reduceExpressions = exprs: - let - cond = { - "and" = x: y: x && y; - "or" = x: y: x || y; - }; - reduceExpressionsFun = acc: v: ( - if builtins.typeOf v == "set" then - ( - if v.type == "value" then - ( - acc // { - value = cond."${acc.cond}" acc.value v.value; - } - ) else if v.type == "bool" then - ( - acc // { - cond = v.value; - } - ) else throw "Unsupported type" - ) else if builtins.typeOf v == "list" then - ( - let - ret = builtins.foldl' - reduceExpressionsFun - { - value = true; - cond = "and"; - } - v; - in - acc // { - value = cond."${acc.cond}" acc.value ret.value; - } - ) else throw "Unsupported type" - ); - in - ( - builtins.foldl' - reduceExpressionsFun - { - value = true; - cond = "and"; - } - exprs - ).value; -in -e: builtins.foldl' (acc: v: v acc) e [ - findSubExpressions - parseExpressions - transformExpressions - evalExpressions - reduceExpressions -] diff --git a/tests/in-list/test_sqlalchemy.py b/tests/in-list/inlist.py similarity index 100% rename from tests/in-list/test_sqlalchemy.py rename to tests/in-list/inlist.py diff --git a/tests/in-list/pyproject.toml b/tests/in-list/pyproject.toml index c020526..ddfcf33 100644 --- a/tests/in-list/pyproject.toml +++ b/tests/in-list/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "test_sqlalchemy" +name = "inlist" version = "0.1.0" description = "poetry2nix test" authors = ["Your Name "] From 3a448dd440665e069bc70ca4939d8b7bed4e2475 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Oct 2023 16:06:56 +1300 Subject: [PATCH 4/4] Use normalizePackageName from pyproject.nix --- default.nix | 6 +++++- editable.nix | 3 ++- lib.nix | 13 ------------- mk-poetry-dep.nix | 3 ++- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/default.nix b/default.nix index ab262ad..ac38de9 100644 --- a/default.nix +++ b/default.nix @@ -3,10 +3,14 @@ , poetryLib ? import ./lib.nix { inherit lib pkgs; inherit (pkgs) stdenv; } }: let - inherit (poetryLib) isCompatible readTOML normalizePackageName normalizePackageSet; + inherit (poetryLib) isCompatible readTOML; pyproject-nix = import ./vendor/pyproject.nix { inherit lib; }; + # Name normalization + inherit (pyproject-nix.pypa) normalizePackageName; + normalizePackageSet = lib.attrsets.mapAttrs' (name: value: lib.attrsets.nameValuePair (normalizePackageName name) value); + # Map SPDX identifiers to license names spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses))); # Get license by id falling back to input string diff --git a/editable.nix b/editable.nix index de33966..5897212 100644 --- a/editable.nix +++ b/editable.nix @@ -3,9 +3,10 @@ , pyProject , python , editablePackageSources +, pyproject-nix }: let - name = poetryLib.normalizePackageName pyProject.tool.poetry.name; + name = pyproject-nix.pypa.normalizePackageName pyProject.tool.poetry.name; # Just enough standard PKG-INFO fields for an editable installation pkgInfoFields = { diff --git a/lib.nix b/lib.nix index 058fab8..6499b9c 100644 --- a/lib.nix +++ b/lib.nix @@ -8,17 +8,6 @@ let genList (i: if i == idx then value else (builtins.elemAt list i)) (length list) ); - # Normalize package names as per PEP 503 - normalizePackageName = name: - let - parts = builtins.split "[-_.]+" name; - partsWithoutSeparator = builtins.filter (x: builtins.typeOf x == "string") parts; - in - lib.strings.toLower (lib.strings.concatStringsSep "-" partsWithoutSeparator); - - # Normalize an entire attrset of packages - normalizePackageSet = lib.attrsets.mapAttrs' (name: value: lib.attrsets.nameValuePair (normalizePackageName name) value); - # Get a full semver pythonVersion from a python derivation getPythonVersion = python: let @@ -242,8 +231,6 @@ in getBuildSystemPkgs satisfiesSemver cleanPythonSources - normalizePackageName - normalizePackageSet getPythonVersion getTargetMachine ; diff --git a/mk-poetry-dep.nix b/mk-poetry-dep.nix index cbbb3f0..7534589 100644 --- a/mk-poetry-dep.nix +++ b/mk-poetry-dep.nix @@ -29,7 +29,8 @@ pythonPackages.callPackage }@args: let inherit (python) stdenv; - inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi normalizePackageName; + inherit (pyproject-nix.pypa) normalizePackageName; + inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi; inherit (import ./pep425.nix { inherit lib poetryLib python stdenv;