mirror of
https://github.com/vale981/poetry2nix
synced 2025-03-04 16:51:40 -05:00
Refactoring
- Extract utility functions to ./lib.nix - Extract buildPythonPackage to ./mk-poetry-dep.nix - Replace some functions with existing builtin functions - Use lib.partition for python package splitting - Drop package filtering on the top-level and leave it to pep425.nix
This commit is contained in:
Tobias Pflug
Andreas Rammhold
parent
29c93b1dc2
commit
84f27ee31d
7 changed files with 198 additions and 191 deletions
213
default.nix
213
default.nix
|
|
@ -1,12 +1,18 @@
|
|||
{ pkgs ? import <nixpkgs> {}
|
||||
, lib ? pkgs.lib
|
||||
, poetry ? null
|
||||
,
|
||||
, poetryLib ? import ./lib.nix { inherit lib pkgs; }
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (poetryLib) isCompatible readTOML;
|
||||
|
||||
importTOML = path: builtins.fromTOML (builtins.readFile path);
|
||||
defaultPoetryOverrides = import ./overrides.nix { inherit pkgs; };
|
||||
|
||||
mkEvalPep508 = import ./pep508.nix {
|
||||
inherit lib;
|
||||
stdenv = pkgs.stdenv;
|
||||
};
|
||||
|
||||
getAttrDefault = attribute: set: default: (
|
||||
if builtins.hasAttr attribute set
|
||||
|
|
@ -14,94 +20,6 @@ let
|
|||
else default
|
||||
);
|
||||
|
||||
# Fetch the artifacts from the PyPI index. Since we get all
|
||||
# info we need from the lock file we don't use nixpkgs' fetchPyPi
|
||||
# as it modifies casing while not providing anything we don't already
|
||||
# have.
|
||||
#
|
||||
# Args:
|
||||
# pname: package name
|
||||
# file: filename including extension
|
||||
# hash: SRI hash
|
||||
# kind: Language implementation and version tag https://www.python.org/dev/peps/pep-0427/#file-name-convention
|
||||
fetchFromPypi = lib.makeOverridable (
|
||||
{ pname, file, hash, kind }:
|
||||
pkgs.fetchurl {
|
||||
url = "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}";
|
||||
inherit hash;
|
||||
}
|
||||
);
|
||||
|
||||
getAttrPath = attrPath: set: (
|
||||
builtins.foldl'
|
||||
(acc: v: if builtins.typeOf acc == "set" && builtins.hasAttr v acc then acc."${v}" else null)
|
||||
set (lib.splitString "." attrPath)
|
||||
);
|
||||
|
||||
satisfiesSemver = (import ./semver.nix { inherit lib; }).satisfies;
|
||||
|
||||
# Check Python version is compatible with package
|
||||
isCompatible = pythonVersion: pythonVersions: let
|
||||
operators = {
|
||||
"||" = cond1: cond2: cond1 || cond2;
|
||||
"," = cond1: cond2: cond1 && cond2; # , means &&
|
||||
};
|
||||
tokens = builtins.filter (x: x != "") (builtins.split "(,|\\|\\|)" pythonVersions);
|
||||
in
|
||||
(
|
||||
builtins.foldl' (
|
||||
acc: v: let
|
||||
isOperator = builtins.typeOf v == "list";
|
||||
operator = if isOperator then (builtins.elemAt v 0) else acc.operator;
|
||||
in
|
||||
if isOperator then (acc // { inherit operator; }) else {
|
||||
inherit operator;
|
||||
state = operators."${operator}" acc.state (satisfiesSemver pythonVersion v);
|
||||
}
|
||||
)
|
||||
{
|
||||
operator = ",";
|
||||
state = true;
|
||||
}
|
||||
tokens
|
||||
).state;
|
||||
|
||||
extensions = pkgs.lib.importJSON ./extensions.json;
|
||||
getExtension = filename: builtins.elemAt
|
||||
(builtins.filter (ext: builtins.match "^.*\.${ext}" filename != null) extensions)
|
||||
0;
|
||||
supportedRe = ("^.*?(" + builtins.concatStringsSep "|" extensions + ")");
|
||||
fileSupported = fname: builtins.match supportedRe fname != null;
|
||||
|
||||
defaultPoetryOverrides = import ./overrides.nix { inherit pkgs; };
|
||||
|
||||
isBdist = f: builtins.match "^.*?whl$" f.file != null;
|
||||
isSdist = f: ! isBdist f;
|
||||
|
||||
mkEvalPep508 = import ./pep508.nix {
|
||||
inherit lib;
|
||||
stdenv = pkgs.stdenv;
|
||||
};
|
||||
|
||||
selectWhl = (
|
||||
import ./pep425.nix {
|
||||
inherit lib;
|
||||
inherit (pkgs) stdenv;
|
||||
python = pkgs.python3;
|
||||
}
|
||||
).select;
|
||||
|
||||
#
|
||||
# Returns the appropriate manylinux dependencies and string representation for the file specified
|
||||
#
|
||||
getManyLinuxDeps = f:
|
||||
let
|
||||
ml = pkgs.pythonManylinuxPackages;
|
||||
in
|
||||
if lib.strings.hasInfix "manylinux1" f then { pkg = [ ml.manylinux1 ]; str = "1"; }
|
||||
else if lib.strings.hasInfix "manylinux2010" f then { pkg = [ ml.manylinux2010 ]; str = "2010"; }
|
||||
else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; }
|
||||
else { pkg = []; str = null; };
|
||||
|
||||
mkPoetryPackage =
|
||||
{ src
|
||||
|
|
@ -112,10 +30,9 @@ let
|
|||
, python ? pkgs.python3
|
||||
, ...
|
||||
}@attrs: let
|
||||
pyProject = importTOML pyproject;
|
||||
poetryLock = importTOML poetrylock;
|
||||
|
||||
files = getAttrDefault "files" (getAttrDefault "metadata" poetryLock {}) {};
|
||||
pyProject = readTOML pyproject;
|
||||
poetryLock = readTOML poetrylock;
|
||||
lockFiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock;
|
||||
|
||||
specialAttrs = [ "pyproject" "poetrylock" "overrides" ];
|
||||
passedAttrs = builtins.removeAttrs attrs specialAttrs;
|
||||
|
|
@ -130,93 +47,38 @@ let
|
|||
# closure as python can only ever have one version of a dependency
|
||||
py = let
|
||||
packageOverrides = self: super: let
|
||||
getDep = depName: if builtins.hasAttr depName self then self."${depName}" else null;
|
||||
|
||||
mkPoetryDep = pkgMeta: let
|
||||
pkgFiles = let
|
||||
all = getAttrDefault pkgMeta.name files [];
|
||||
in
|
||||
builtins.filter (f: fileSupported f.file) all;
|
||||
|
||||
filterFile = fname: builtins.match ("^.*" + builtins.replaceStrings [ "." ] [ "\\." ] pkgMeta.version + ".*$") fname != null;
|
||||
filteredFiles = builtins.filter (f: filterFile f.file) pkgFiles;
|
||||
|
||||
binaryDist = selectWhl filteredFiles;
|
||||
sourceDist = builtins.filter isSdist filteredFiles;
|
||||
file = if (builtins.length sourceDist) > 0 then builtins.head sourceDist else builtins.head binaryDist;
|
||||
|
||||
format =
|
||||
if isBdist file
|
||||
then "wheel"
|
||||
else "setuptools";
|
||||
|
||||
in
|
||||
self.buildPythonPackage {
|
||||
pname = pkgMeta.name;
|
||||
version = pkgMeta.version;
|
||||
|
||||
doCheck = false; # We never get development deps
|
||||
dontStrip = true;
|
||||
|
||||
inherit format;
|
||||
|
||||
nativeBuildInputs = [ pkgs.autoPatchelfHook ];
|
||||
buildInputs = (getManyLinuxDeps file.file).pkg;
|
||||
NIX_PYTHON_MANYLINUX = (getManyLinuxDeps file.file).str;
|
||||
|
||||
propagatedBuildInputs = let
|
||||
depAttrs = getAttrDefault "dependencies" pkgMeta {};
|
||||
# Some dependencies like django gets the attribute name django
|
||||
# but dependencies try to access Django
|
||||
dependencies = builtins.map (d: lib.toLower d) (builtins.attrNames depAttrs);
|
||||
in
|
||||
builtins.map getDep dependencies;
|
||||
|
||||
meta = {
|
||||
broken = ! isCompatible python.version pkgMeta.python-versions;
|
||||
license = [];
|
||||
};
|
||||
|
||||
src = fetchFromPypi {
|
||||
pname = pkgMeta.name;
|
||||
inherit (file) file hash;
|
||||
# We need to retrieve kind from the interpreter and the filename of the package
|
||||
# Interpreters should declare what wheel types they're compatible with (python type + ABI)
|
||||
# Here we can then choose a file based on that info.
|
||||
kind = if format == "setuptools" then "source" else (builtins.elemAt (lib.strings.splitString "-" file.file) 2);
|
||||
};
|
||||
};
|
||||
getDep = depName: if builtins.hasAttr depName self then self."${depName}" else throw "foo";
|
||||
|
||||
# Filter packages by their PEP508 markers
|
||||
pkgsWithFilter = builtins.map (
|
||||
pkgMeta: let
|
||||
f = if builtins.hasAttr "marker" pkgMeta then (!evalPep508 pkgMeta.marker) else false;
|
||||
in
|
||||
pkgMeta // { p2nixFiltered = f; }
|
||||
) poetryLock.package;
|
||||
partitions = let
|
||||
supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true;
|
||||
in
|
||||
lib.partition supportsPythonVersion poetryLock.package;
|
||||
|
||||
lockPkgs = builtins.map (
|
||||
pkgMeta: {
|
||||
name = pkgMeta.name;
|
||||
value = let
|
||||
drv = mkPoetryDep pkgMeta;
|
||||
override = getAttrDefault pkgMeta.name overrides (_: _: drv: drv);
|
||||
in
|
||||
if drv != null then (override self super drv) else null;
|
||||
}
|
||||
) (builtins.filter (pkgMeta: !pkgMeta.p2nixFiltered) pkgsWithFilter);
|
||||
compatible = partitions.right;
|
||||
incompatible = partitions.wrong;
|
||||
|
||||
# Null out any filtered packages, we don't want python.pkgs from nixpkgs
|
||||
nulledPkgs = (
|
||||
builtins.listToAttrs
|
||||
(
|
||||
builtins.map (x: { name = x.name; value = null; })
|
||||
(builtins.filter (pkgMeta: pkgMeta.p2nixFiltered) pkgsWithFilter)
|
||||
)
|
||||
lockPkgs = builtins.listToAttrs (
|
||||
builtins.map (
|
||||
pkgMeta: rec {
|
||||
name = pkgMeta.name;
|
||||
value = let
|
||||
drv = self.mkPoetryDep (pkgMeta // { files = lockFiles.${name}; });
|
||||
override = getAttrDefault pkgMeta.name overrides (_: _: drv: drv);
|
||||
in
|
||||
override self super drv;
|
||||
}
|
||||
) compatible
|
||||
);
|
||||
|
||||
# Null out any filtered packages, we don't want python.pkgs from nixpkgs
|
||||
nulledPkgs = builtins.listToAttrs (builtins.map (x: { name = x.name; value = null; }) incompatible);
|
||||
in
|
||||
nulledPkgs // builtins.listToAttrs lockPkgs;
|
||||
{
|
||||
mkPoetryDep = self.callPackage ./mk-poetry-dep.nix {
|
||||
inherit pkgs lib python poetryLib;
|
||||
};
|
||||
} // nulledPkgs // lockPkgs;
|
||||
in
|
||||
python.override { inherit packageOverrides; self = py; };
|
||||
pythonPackages = py.pkgs;
|
||||
|
|
@ -237,8 +99,7 @@ let
|
|||
};
|
||||
|
||||
getBuildSystemPkgs = let
|
||||
buildSystem = getAttrPath
|
||||
"build-system.build-backend" pyProject;
|
||||
buildSystem = lib.getAttrFromPath [ "build-system" "build-backend" ] pyProject;
|
||||
in
|
||||
knownBuildSystems.${buildSystem} or (throw "unsupported build system ${buildSystem}");
|
||||
in
|
||||
|
|
|
|||
67
lib.nix
Normal file
67
lib.nix
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
{ lib, pkgs }:
|
||||
let
|
||||
inherit (import ./semver.nix { inherit lib; }) satisfiesSemver;
|
||||
|
||||
# Returns true if pythonVersion matches with the expression in pythonVersions
|
||||
isCompatible = pythonVersion: pythonVersions:
|
||||
let
|
||||
operators = {
|
||||
"||" = cond1: cond2: cond1 || cond2;
|
||||
"," = cond1: cond2: cond1 && cond2; # , means &&
|
||||
};
|
||||
# split string at "," and "||"
|
||||
tokens = builtins.filter (x: x != "") (builtins.split "(,|\\|\\|)" pythonVersions);
|
||||
combine = acc: v:
|
||||
let
|
||||
isOperator = builtins.typeOf v == "list";
|
||||
operator = if isOperator then (builtins.elemAt v 0) else acc.operator;
|
||||
in
|
||||
if isOperator then (acc // { inherit operator; }) else {
|
||||
inherit operator;
|
||||
state = operators."${operator}" acc.state (satisfiesSemver pythonVersion v);
|
||||
};
|
||||
initial = { operator = ","; state = true; };
|
||||
in
|
||||
(builtins.foldl' combine initial tokens).state;
|
||||
|
||||
readTOML = path: builtins.fromTOML (builtins.readFile path);
|
||||
|
||||
#
|
||||
# Returns the appropriate manylinux dependencies and string representation for the file specified
|
||||
#
|
||||
getManyLinuxDeps = f:
|
||||
let
|
||||
ml = pkgs.pythonManylinuxPackages;
|
||||
in
|
||||
if lib.strings.hasInfix "manylinux1" f then { pkg = [ ml.manylinux1 ]; str = "1"; }
|
||||
else if lib.strings.hasInfix "manylinux2010" f then { pkg = [ ml.manylinux2010 ]; str = "2010"; }
|
||||
else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; }
|
||||
else { pkg = []; str = null; };
|
||||
|
||||
# Fetch the artifacts from the PyPI index. Since we get all
|
||||
# info we need from the lock file we don't use nixpkgs' fetchPyPi
|
||||
# as it modifies casing while not providing anything we don't already
|
||||
# have.
|
||||
#
|
||||
# Args:
|
||||
# pname: package name
|
||||
# file: filename including extension
|
||||
# hash: SRI hash
|
||||
# kind: Language implementation and version tag https://www.python.org/dev/peps/pep-0427/#file-name-convention
|
||||
fetchFromPypi = lib.makeOverridable (
|
||||
{ pname, file, hash, kind }:
|
||||
pkgs.fetchurl {
|
||||
url = "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}";
|
||||
inherit hash;
|
||||
}
|
||||
);
|
||||
|
||||
in
|
||||
{
|
||||
inherit
|
||||
fetchFromPypi
|
||||
getManyLinuxDeps
|
||||
isCompatible
|
||||
readTOML
|
||||
;
|
||||
}
|
||||
79
mk-poetry-dep.nix
Normal file
79
mk-poetry-dep.nix
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
{ autoPatchelfHook
|
||||
, pkgs
|
||||
, lib
|
||||
, python
|
||||
, buildPythonPackage
|
||||
, pythonPackages
|
||||
, poetryLib
|
||||
}:
|
||||
{ name
|
||||
, version
|
||||
, files
|
||||
, dependencies ? {}
|
||||
, python-versions
|
||||
, supportedExtensions ? lib.importJSON ./extensions.json
|
||||
, ...
|
||||
}: let
|
||||
|
||||
inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi;
|
||||
|
||||
inherit (import ./pep425.nix {
|
||||
inherit lib python;
|
||||
inherit (pkgs) stdenv;
|
||||
}) selectWheel;
|
||||
|
||||
fileCandidates = let
|
||||
supportedRegex = ("^.*?(" + builtins.concatStringsSep "|" supportedExtensions + ")");
|
||||
matchesVersion = fname: builtins.match ("^.*" + builtins.replaceStrings [ "." ] [ "\\." ] version + ".*$") fname != null;
|
||||
hasSupportedExtension = fname: builtins.match supportedRegex fname != null;
|
||||
in
|
||||
builtins.filter (f: matchesVersion f.file && hasSupportedExtension f.file) files;
|
||||
|
||||
fileInfo = let
|
||||
isBdist = f: lib.strings.hasSuffix "whl" f.file;
|
||||
isSdist = f: ! isBdist f;
|
||||
binaryDist = selectWheel fileCandidates;
|
||||
sourceDist = builtins.filter isSdist fileCandidates;
|
||||
lockFileEntry = if (builtins.length sourceDist) > 0 then builtins.head sourceDist else builtins.head binaryDist;
|
||||
in
|
||||
rec {
|
||||
inherit (lockFileEntry) file hash;
|
||||
name = file;
|
||||
format = if lib.strings.hasSuffix ".whl" name then "wheel" else "setuptools";
|
||||
kind = if format == "setuptools" then "source" else (builtins.elemAt (lib.strings.splitString "-" name) 2);
|
||||
};
|
||||
|
||||
in
|
||||
buildPythonPackage {
|
||||
pname = name;
|
||||
version = version;
|
||||
|
||||
doCheck = false; # We never get development deps
|
||||
dontStrip = true;
|
||||
format = fileInfo.format;
|
||||
|
||||
nativeBuildInputs = if (getManyLinuxDeps fileInfo.name).str != null then [ autoPatchelfHook ] else [];
|
||||
buildInputs = (getManyLinuxDeps fileInfo.name).pkg;
|
||||
NIX_PYTHON_MANYLINUX = (getManyLinuxDeps fileInfo.name).str;
|
||||
|
||||
propagatedBuildInputs =
|
||||
let
|
||||
# Some dependencies like django gets the attribute name django
|
||||
# but dependencies try to access Django
|
||||
deps = builtins.map (d: lib.toLower d) (builtins.attrNames dependencies);
|
||||
in
|
||||
builtins.map (n: pythonPackages.${n}) deps;
|
||||
|
||||
meta = {
|
||||
broken = ! isCompatible python.version python-versions;
|
||||
license = [];
|
||||
};
|
||||
|
||||
# We need to retrieve kind from the interpreter and the filename of the package
|
||||
# Interpreters should declare what wheel types they're compatible with (python type + ABI)
|
||||
# Here we can then choose a file based on that info.
|
||||
src = fetchFromPypi {
|
||||
pname = name;
|
||||
inherit (fileInfo) file hash kind;
|
||||
};
|
||||
}
|
||||
|
|
@ -57,7 +57,7 @@ let
|
|||
#
|
||||
# Selects the best matching wheel file from a list of files
|
||||
#
|
||||
select = files:
|
||||
selectWheel = files:
|
||||
let
|
||||
filesWithoutSources = (builtins.filter (x: hasSuffix ".whl" x.file) files);
|
||||
|
||||
|
|
@ -97,5 +97,5 @@ let
|
|||
|
||||
in
|
||||
{
|
||||
inherit select;
|
||||
inherit selectWheel;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -71,10 +71,10 @@ let
|
|||
else throw "Constraint \"${constraintStr}\" could not be parsed"
|
||||
);
|
||||
|
||||
satisfies = version: constraint: let
|
||||
satisfiesSemver = version: constraint: let
|
||||
inherit (parseConstraint constraint) op v;
|
||||
in
|
||||
if constraint == "*" then true else operators."${op}" version v;
|
||||
|
||||
in
|
||||
{ inherit satisfies; }
|
||||
{ inherit satisfiesSemver; }
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ let
|
|||
poetryLock = ./poetry.lock;
|
||||
src = lib.cleanSource ./.;
|
||||
};
|
||||
p = pkg.py.withPackages (ps: [ ps.numpy ps.opencv-python ]);
|
||||
p = pkg.python.withPackages (ps: [ ps.numpy ps.opencv-python ]);
|
||||
in
|
||||
runCommand "test" {} ''
|
||||
${p}/bin/python -c "import cv2"
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ lib.debug.runTests {
|
|||
];
|
||||
in
|
||||
{
|
||||
expr = (pep425.select cs);
|
||||
expr = (pep425.selectWheel cs);
|
||||
expected = [ { file = "grpcio-1.25.0-cp27-cp27m-manylinux2010_x86_64.whl"; } ];
|
||||
};
|
||||
|
||||
|
|
@ -22,7 +22,7 @@ lib.debug.runTests {
|
|||
];
|
||||
in
|
||||
{
|
||||
expr = (pep425OSX.select cs);
|
||||
expr = (pep425OSX.selectWheel cs);
|
||||
expected = [ { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_10_x86_64.whl"; } ];
|
||||
};
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ lib.debug.runTests {
|
|||
];
|
||||
in
|
||||
{
|
||||
expr = (pep425Python37.select cs);
|
||||
expr = (pep425Python37.selectWheel cs);
|
||||
expected = [ { file = "grpcio-1.25.0-cp37-cp37m-manylinux2010_x86_64.whl"; } ];
|
||||
};
|
||||
|
||||
|
|
@ -50,7 +50,7 @@ lib.debug.runTests {
|
|||
];
|
||||
in
|
||||
{
|
||||
expr = (pep425OSX.select cs);
|
||||
expr = (pep425OSX.selectWheel cs);
|
||||
expected = [ { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_12_x86_64.whl"; } ];
|
||||
};
|
||||
|
||||
|
|
@ -62,7 +62,7 @@ lib.debug.runTests {
|
|||
];
|
||||
in
|
||||
{
|
||||
expr = (pep425OSX.select cs);
|
||||
expr = (pep425OSX.selectWheel cs);
|
||||
expected = [];
|
||||
};
|
||||
|
||||
|
|
@ -74,7 +74,7 @@ lib.debug.runTests {
|
|||
];
|
||||
in
|
||||
{
|
||||
expr = (pep425.select cs);
|
||||
expr = (pep425.selectWheel cs);
|
||||
expected = [ { file = "grpcio-1.25.0-cp27-cp27m-manylinux1_x86_64.whl"; } ];
|
||||
};
|
||||
|
||||
|
|
@ -86,17 +86,17 @@ lib.debug.runTests {
|
|||
];
|
||||
in
|
||||
{
|
||||
expr = (pep425.select cs);
|
||||
expr = (pep425.selectWheel cs);
|
||||
expected = [];
|
||||
};
|
||||
|
||||
testLinuxEmptyList = {
|
||||
expr = pep425.select [];
|
||||
expr = pep425.selectWheel [];
|
||||
expected = [];
|
||||
};
|
||||
|
||||
testOSXEmptyList = {
|
||||
expr = pep425OSX.select [];
|
||||
expr = pep425OSX.selectWheel [];
|
||||
expected = [];
|
||||
};
|
||||
|
||||
|
|
@ -139,7 +139,7 @@ lib.debug.runTests {
|
|||
];
|
||||
in
|
||||
{
|
||||
expr = pep425.select cs;
|
||||
expr = pep425.selectWheel cs;
|
||||
expected = [ { file = "cffi-1.13.2-cp27-cp27m-manylinux1_x86_64.whl"; } ];
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue