hiro/poetry2nix
1
0
Fork 0
mirror of https://github.com/vale981/poetry2nix synced 2025-03-04 08:41:42 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: big messy refactoring

Browse source 
- normalize package names to match PEP-503
- don't assume build dependencies on setuptools{,-scm}
- automatically generate overrides for setuptools{,-scm}
- pull known build systems directly from nixpkgs
- fix a bunch of mostly unrelated warnings

Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com>
This commit is contained in:
K900 2022-09-29 20:39:37 +03:00
parent 8be90b6f00
commit 33db1f30d0
15 changed files with 16639 additions and 665 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
default.nix
View file

@ -7,7 +7,7 @@ let
# Poetry2nix version
version = "1.31.0";
inherit (poetryLib) isCompatible readTOML moduleName underscorify;
inherit (poetryLib) isCompatible readTOML normalizePackageName normalizePackageSet;
# Map SPDX identifiers to license names
spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses)));
@ -17,6 +17,10 @@ let
# Experimental withPlugins functionality
toPluginAble = (import ./plugins.nix { inherit pkgs lib; }).toPluginAble;
# List of known build systems that are passed through from nixpkgs unmodified
knownBuildSystems = builtins.fromJSON (builtins.readFile ./known-build-systems.json);
nixpkgsBuildSystems = lib.subtractLists [ "poetry" "poetry-core" ] knownBuildSystems;
mkInputAttrs =
{ py
, pyProject
@ -38,7 +42,7 @@ let
(
dep:
let
pkg = py.pkgs."${moduleName dep}";
pkg = py.pkgs."${normalizePackageName dep}";
constraints = depSet.${dep}.python or "";
isCompat = compat constraints;
in
@ -59,7 +63,6 @@ let
buildInputs = mkInput "buildInputs" (if includeBuildSystem then buildSystemPkgs else [ ]);
propagatedBuildInputs = mkInput "propagatedBuildInputs" (
(getDeps pyProject.tool.poetry."dependencies" or { })
++ ([ py.pkgs.setuptools ])
++ (
# >=poetry-1.2.0 dependency groups
if pyProject.tool.poetry.group or { } != { }
@ -128,7 +131,7 @@ lib.makeScope pkgs.newScope (self: {
, editablePackageSources ? { }
, pyProject ? readTOML pyproject
, groups ? [ ]
}@attrs:
}:
let
/* The default list of poetry2nix override overlays */
mkEvalPep508 = import ./pep508.nix {
@ -157,15 +160,7 @@ lib.makeScope pkgs.newScope (self: {
let
lockfiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock;
in
lib.listToAttrs (lib.mapAttrsToList (n: v: { name = moduleName n; value = v; }) lockfiles);
specialAttrs = [
"overrides"
"poetrylock"
"projectDir"
"pwd"
"preferWheels"
];
passedAttrs = builtins.removeAttrs attrs specialAttrs;
lib.listToAttrs (lib.mapAttrsToList (n: v: { name = normalizePackageName n; value = v; }) lockfiles);
evalPep508 = mkEvalPep508 python;
# Filter packages by their PEP508 markers & pyproject interpreter version
@ -183,12 +178,15 @@ lib.makeScope pkgs.newScope (self: {
# closure as python can only ever have one version of a dependency
baseOverlay = self: super:
let
getDep = depName: self.${depName};
lockPkgs = builtins.listToAttrs (
builtins.map
(
pkgMeta: rec {
name = moduleName pkgMeta.name;
pkgMeta:
if builtins.elem pkgMeta.name nixpkgsBuildSystems then {
name = pkgMeta.name;
value = super."${pkgMeta.name}";
} else rec {
name = normalizePackageName pkgMeta.name;
value = self.mkPoetryDep (
pkgMeta // {
inherit pwd preferWheels;
@ -196,17 +194,16 @@ lib.makeScope pkgs.newScope (self: {
files = lockFiles.${name};
pythonPackages = self;
# Packages can be specified with underscores in pyproject.toml; check for
# both possibilities.
sourceSpec = with pyProject.tool.poetry; (
dependencies.${pkgMeta.name} or
dependencies.${underscorify pkgMeta.name} or
dev-dependencies.${pkgMeta.name} or
dev-dependencies.${underscorify pkgMeta.name} or
group.dev.dependencies.${underscorify pkgMeta.name} or # Poetry 1.2.0+
{ }
);
sourceSpec =
let
normalizedName = normalizePackageName pkgMeta.name;
in
with pyProject.tool.poetry; (
(normalizePackageSet dependencies).${normalizedName}
or (normalizePackageSet dev-dependencies).${normalizedName}
or (normalizePackageSet group.dev.dependencies).${normalizedName} # Poetry 1.2.0+
or { }
);
}
);
}
@ -255,7 +252,7 @@ lib.makeScope pkgs.newScope (self: {
super)
# Null out any filtered packages, we don't want python.pkgs from nixpkgs
(self: super: builtins.listToAttrs (builtins.map (x: { name = moduleName x.name; value = null; }) incompatible))
(self: super: builtins.listToAttrs (builtins.map (x: { name = normalizePackageName x.name; value = null; }) incompatible))
# Create poetry2nix layer
baseOverlay
@ -303,7 +300,7 @@ lib.makeScope pkgs.newScope (self: {
, groups ? [ "dev" ]
}:
let
inherit (lib) elem hasAttr;
inherit (lib) hasAttr;
pyProject = readTOML pyproject;
@ -395,7 +392,7 @@ lib.makeScope pkgs.newScope (self: {
py.pkgs.removeGitDependenciesHook
];
} // {
pname = moduleName pyProject.tool.poetry.name;
pname = normalizePackageName pyProject.tool.poetry.name;
version = pyProject.tool.poetry.version;
inherit src;
@ -477,7 +474,7 @@ lib.makeScope pkgs.newScope (self: {
Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function
*/
defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib; });
defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib poetryLib; });
/*
Convenience functions for specifying overlays with or without the poerty2nix default overrides

2
editable.nix
View file

@ -6,7 +6,7 @@
, editablePackageSources
}:
let
name = poetryLib.moduleName pyProject.tool.poetry.name;
name = poetryLib.normalizePackageName pyProject.tool.poetry.name;
# Just enough standard PKG-INFO fields for an editable installation
pkgInfoFields = {

12
known-build-systems.json Normal file
View file

@ -0,0 +1,12 @@
[
"poetry",
"poetry-core",
"flit",
"flit-core",
"pbr",
"flitBuildHook",
"cython",
"hatchling",
"setuptools",
"setuptools-scm"
]

18
lib.nix
View file

@ -8,12 +8,16 @@ let
genList (i: if i == idx then value else (builtins.elemAt list i)) (length list)
);
# Do some canonicalisation of module names
moduleName = name: lib.toLower (lib.replaceStrings [ "_" "." ] [ "-" "-" ] name);
# Normalize package names as per PEP 503
normalizePackageName = name:
let
parts = builtins.split "[-_.]+" name;
partsWithoutSeparator = builtins.filter (x: builtins.typeOf x == "string") parts;
in
lib.strings.toLower (lib.strings.concatStringsSep "-" partsWithoutSeparator);
# For some reason, poetry replaces underscores with dashes in module
# names, this has to be reversed sometimes
underscorify = name: (lib.replaceStrings [ "-" ] [ "_" ] name);
# Normalize an entire attrset of packages
normalizePackageSet = lib.attrsets.mapAttrs' (name: value: lib.attrsets.nameValuePair (normalizePackageName name) value);
# Get a full semver pythonVersion from a python derivation
getPythonVersion = python:
@ -237,8 +241,8 @@ in
getBuildSystemPkgs
satisfiesSemver
cleanPythonSources
moduleName
underscorify
normalizePackageName
normalizePackageSet
getPythonVersion
getTargetMachine
;

24
mk-poetry-dep.nix
View file

@ -26,7 +26,7 @@ pythonPackages.callPackage
}@args:
let
inherit (python) stdenv;
inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi moduleName;
inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi normalizePackageName;
inherit (import ./pep425.nix {
inherit lib poetryLib python stdenv;
@ -88,23 +88,10 @@ pythonPackages.callPackage
else (builtins.elemAt (lib.strings.splitString "-" name) 2);
};
# Prevent infinite recursion
skipSetupToolsSCM = [
"setuptools_scm"
"setuptools-scm"
"toml" # Toml is an extra for setuptools-scm
"tomli" # tomli is an extra for later versions of setuptools-scm
"flit-core"
"packaging"
"six"
"pyparsing"
"typing-extensions"
];
baseBuildInputs = lib.optional (! lib.elem name skipSetupToolsSCM) pythonPackages.setuptools-scm;
format = if isDirectory || isGit || isUrl then "pyproject" else fileInfo.format;
in
buildPythonPackage {
pname = moduleName name;
pname = normalizePackageName name;
version = version;
inherit format;
@ -124,10 +111,9 @@ pythonPackages.callPackage
];
buildInputs = (
baseBuildInputs
++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pythonPackages.setuptools
++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg
lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg
++ lib.optional isDirectory buildSystemPkgs
++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pythonPackages.setuptools
);
propagatedBuildInputs =
@ -149,7 +135,7 @@ pythonPackages.callPackage
);
depAttrs = lib.attrNames deps;
in
builtins.map (n: pythonPackages.${moduleName n}) depAttrs;
builtins.map (n: pythonPackages.${normalizePackageName n}) depAttrs;
meta = {
broken = ! isCompatible (poetryLib.getPythonVersion python) python-versions;

17058
overrides/build-systems.json
View file

File diff suppressed because it is too large Load diff

40
overrides/default.nix
View file

@ -1,6 +1,7 @@
{ pkgs ? import <nixpkgs> { }
, lib ? pkgs.lib
, stdenv ? pkgs.stdenv
, poetryLib
}:
let
@ -38,6 +39,7 @@ let
# Flit only works on Python3
if (attr == "flit-core" || attr == "flit" || attr == "hatchling") && !self.isPy3k then drv
else if drv == null then null
else if drv ? overridePythonAttrs == false then drv
else
drv.overridePythonAttrs (
old:
@ -57,6 +59,8 @@ let
in
lib.composeManyExtensions [
# normalize all the names
(self: super: poetryLib.normalizePackageSet super)
# NixOps
(self: super:
@ -81,22 +85,11 @@ lib.composeManyExtensions [
systems)
buildSystems)
# Build systems with conditionals
(self: super: {
platformdirs =
if lib.versionAtLeast super.platformdirs.version "2.5.2"
then addBuildSystem { inherit self; drv = super.platformdirs; attr = "hatchling"; extraAttrs = [ "hatch-vcs" ]; }
else super.platformdirs;
})
# Build fixes
(self: super:
let
inherit (self.python) stdenv;
inherit (pkgs.buildPackages) pkg-config;
inherit (pkgs) buildPackages;
pyBuildPackages = self.python.pythonForBuild.pkgs;
selectQt5 = version:
@ -109,7 +102,7 @@ lib.composeManyExtensions [
{
automat = super.automat.overridePythonAttrs (
old: rec {
old: {
propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.m2r ];
}
);
@ -153,7 +146,7 @@ lib.composeManyExtensions [
);
argcomplete = super.argcomplete.overridePythonAttrs (
old: rec {
old: {
buildInputs = (old.buildInputs or [ ]) ++ [ self.importlib-metadata ];
}
);
@ -165,7 +158,7 @@ lib.composeManyExtensions [
);
astroid = super.astroid.overridePythonAttrs (
old: rec {
old: {
buildInputs = (old.buildInputs or [ ]) ++ [ self.pytest-runner ];
}
);
@ -657,6 +650,11 @@ lib.composeManyExtensions [
outputs = [ "out" "dev" ];
});
gunicorn = super.gunicorn.overridePythonAttrs (old: {
# actually needs setuptools as a runtime dependency
propagatedBuildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools ];
});
h3 = super.h3.overridePythonAttrs (
old: {
preBuild = (old.preBuild or "") + ''
@ -877,7 +875,7 @@ lib.composeManyExtensions [
else super.jsonschema;
jupyter = super.jupyter.overridePythonAttrs (
old: rec {
old: {
# jupyter is a meta-package. Everything relevant comes from the
# dependencies. It does however have a jupyter.py file that conflicts
# with jupyter-core so this meta solves this conflict.
@ -890,7 +888,7 @@ lib.composeManyExtensions [
});
jupyterlab-widgets = super.jupyterlab-widgets.overridePythonAttrs (
old: rec {
old: {
buildInputs = (old.buildInputs or [ ]) ++ [ self.jupyter-packaging ];
}
);
@ -1131,11 +1129,11 @@ lib.composeManyExtensions [
excludes = [ "pyproject.toml" ];
})
];
buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools-scm-git-archive ];
buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools self.setuptools-scm self.setuptools-scm-git-archive ];
}
)) else
super.molecule.overridePythonAttrs (old: {
buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools-scm-git-archive ];
buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools self.setuptools-scm self.setuptools-scm-git-archive ];
});
mpi4py = super.mpi4py.overridePythonAttrs (
@ -1300,7 +1298,7 @@ lib.composeManyExtensions [
);
openexr = super.openexr.overridePythonAttrs (
old: rec {
old: {
buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.openexr pkgs.ilmbase ];
NIX_CFLAGS_COMPILE = [ "-I${pkgs.openexr.dev}/include/OpenEXR" "-I${pkgs.ilmbase.dev}/include/OpenEXR" ];
}
@ -1385,7 +1383,7 @@ lib.composeManyExtensions [
});
parsel = super.parsel.overridePythonAttrs (
old: rec {
old: {
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.pytest-runner ];
}
);
@ -1934,7 +1932,7 @@ lib.composeManyExtensions [
);
rockset = super.rockset.overridePythonAttrs (
old: rec {
old: {
postPatch = ''
cp ./setup_rockset.py ./setup.py
'';

1
tests/default.nix
View file

@ -121,5 +121,6 @@ builtins.removeAttrs
# Once this is available in 19.09 and unstable we can re-enable the manylinux test
manylinux = callTest ./manylinux { };
shapely = callTest ./shapely { };
setuptools = callTest ./setuptools { };
}
skipTests

2
tests/extras/poetry.lock generated
View file

@ -327,7 +327,7 @@ python-versions = "*"
[metadata]
lock-version = "1.1"
python-versions = "^3.6"
content-hash = "5a83192fb095e50ac54daf8b7d6c63abf824fdd5d87458b93bfe88b0e201c0c6"
content-hash = "29715f340ec421fd9e3d4ec49f4363e9129714832fb17282c8b8dba9b3833db7"
[metadata.files]
boto3-stubs = [

2
tests/extras/pyproject.toml
View file

@ -6,7 +6,7 @@ authors = ["Your Name <you@example.com>"]
[tool.poetry.dependencies]
python = "^3.6"
setuptools-scm = { version = "^3.4", extras = ["toml"] }
setuptools-scm = { version = ">=3.4", extras = ["toml"] }
# Extra with dash in the name should work
boto3-stubs = {extras = ["cognito-idp"], version = "^1.17.82"}

8
tests/setuptools/default.nix Normal file
View file

@ -0,0 +1,8 @@
{ lib, poetry2nix, python3 }:
poetry2nix.mkPoetryApplication {
python = python3;
pyproject = ./pyproject.toml;
poetrylock = ./poetry.lock;
src = lib.cleanSource ./.;
}

23
tests/setuptools/poetry.lock generated Normal file
View file

@ -0,0 +1,23 @@
[[package]]
name = "setuptools"
version = "65.4.1"
description = "Easily download, build, install, upgrade, and uninstall Python packages"
category = "main"
optional = false
python-versions = ">=3.7"
[package.extras]
docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
[metadata]
lock-version = "1.1"
python-versions = "^3.10"
content-hash = "bbc2fde99c9aade70aa71c355e2067e25e7d3993814d132cf6a9d8acbb3f251e"
[metadata.files]
setuptools = [
{file = "setuptools-65.4.1-py3-none-any.whl", hash = "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012"},
{file = "setuptools-65.4.1.tar.gz", hash = "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"},
]

13
tests/setuptools/pyproject.toml Normal file
View file

@ -0,0 +1,13 @@
[tool.poetry]
name = "setuptools_test"
version = "0.1.0"
description = ""
authors = ["Your Name <you@example.com>"]
[tool.poetry.dependencies]
python = "^3.10"
setuptools = "*"
[build-system]
requires = ["poetry-core>=1.1"]
build-backend = "poetry.core.masonry.api"

0
tests/setuptools/setuptools_test/__init__.py Normal file
View file

42
tools/find-build-systems.py
View file

@ -2,11 +2,13 @@
from concurrent.futures import ThreadPoolExecutor
from itertools import chain
from posix import cpu_count
import re
from typing import (
Dict,
List,
Set,
)
from pathlib import Path
import subprocess
import pynixutil
import tempfile
@ -21,16 +23,8 @@ import sys
# All known PEP-517 (or otherwise) build systems
BUILD_SYSTEMS = [
"poetry",
"poetry-core",
"flit",
"flit-core",
"pbr",
"flitBuildHook",
"cython",
"hatchling",
]
with (Path(__file__).parent.parent / "known-build-systems.json").open() as _fd:
BUILD_SYSTEMS = json.load(_fd)
# Skip these attributes as they have more complex conditions manually
@ -40,10 +34,16 @@ SKIP_ATTRS = {
"packaging",
"poetry",
"flitBuildHook",
"jsonschema",
"platformdirs",
"traitlets",
}
def normalize(name):
return re.sub(r"[-_.]+", "-", name).lower()
def find_known_systems() -> Dict[str, str]:
"""Create a map from attribute to drvPath for known build systems"""
@ -130,15 +130,31 @@ def get_build_systems(known_systems) -> Dict[str, List[str]]:
return {attr: systems for attr, systems in build_systems.items() if systems}
BLOCKLIST = {"poetry", "poetry-core"}
def merge_systems(s):
simple = {i for i in s if isinstance(i, str)}
complex = [i for i in s if isinstance(i, dict)]
complex_names = {i["buildSystem"] for i in complex}
new_simple = simple - complex_names
return complex + sorted(list(new_simple))
def merge(prev_content, new_content):
content = {}
for attr, systems in chain(prev_content.items(), new_content.items()):
s = content.setdefault(attr, set())
attr = normalize(attr)
s = content.setdefault(attr, [])
for system in systems:
s.add(system)
s.append(system)
# Return with sorted data
return {attr: sorted(content[attr]) for attr in sorted(content.keys())}
return {
attr: merge_systems(content[attr])
for attr in sorted(content.keys())
if attr not in BLOCKLIST
}
def main():