feat: big messy refactoring

- normalize package names to match PEP-503
- don't assume build dependencies on setuptools{,-scm}
- automatically generate overrides for setuptools{,-scm}
- pull known build systems directly from nixpkgs
- fix a bunch of mostly unrelated warnings

Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com>

default.nix

@@ -7,7 +7,7 @@ let
   # Poetry2nix version
   version = "1.31.0";
 
-  inherit (poetryLib) isCompatible readTOML moduleName underscorify;
+  inherit (poetryLib) isCompatible readTOML normalizePackageName normalizePackageSet;
 
   # Map SPDX identifiers to license names
   spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses)));
@@ -17,6 +17,10 @@ let
   # Experimental withPlugins functionality
   toPluginAble = (import ./plugins.nix { inherit pkgs lib; }).toPluginAble;
 
+  # List of known build systems that are passed through from nixpkgs unmodified
+  knownBuildSystems = builtins.fromJSON (builtins.readFile ./known-build-systems.json);
+  nixpkgsBuildSystems = lib.subtractLists [ "poetry" "poetry-core" ] knownBuildSystems;
+
   mkInputAttrs =
     { py
     , pyProject
@@ -38,7 +42,7 @@ let
             (
               dep:
               let
-                pkg = py.pkgs."${moduleName dep}";
+                pkg = py.pkgs."${normalizePackageName dep}";
                 constraints = depSet.${dep}.python or "";
                 isCompat = compat constraints;
               in
@@ -59,7 +63,6 @@ let
       buildInputs = mkInput "buildInputs" (if includeBuildSystem then buildSystemPkgs else [ ]);
       propagatedBuildInputs = mkInput "propagatedBuildInputs" (
         (getDeps pyProject.tool.poetry."dependencies" or { })
-        ++ ([ py.pkgs.setuptools ])
         ++ (
           # >=poetry-1.2.0 dependency groups
           if pyProject.tool.poetry.group or { } != { }
@@ -128,7 +131,7 @@ lib.makeScope pkgs.newScope (self: {
     , editablePackageSources ? { }
     , pyProject ? readTOML pyproject
     , groups ? [ ]
-    }@attrs:
+    }:
     let
       /* The default list of poetry2nix override overlays */
       mkEvalPep508 = import ./pep508.nix {
@@ -157,15 +160,7 @@ lib.makeScope pkgs.newScope (self: {
         let
           lockfiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock;
         in
-        lib.listToAttrs (lib.mapAttrsToList (n: v: { name = moduleName n; value = v; }) lockfiles);
-      specialAttrs = [
-        "overrides"
-        "poetrylock"
-        "projectDir"
-        "pwd"
-        "preferWheels"
-      ];
-      passedAttrs = builtins.removeAttrs attrs specialAttrs;
+        lib.listToAttrs (lib.mapAttrsToList (n: v: { name = normalizePackageName n; value = v; }) lockfiles);
       evalPep508 = mkEvalPep508 python;
 
       # Filter packages by their PEP508 markers & pyproject interpreter version
@@ -183,12 +178,15 @@ lib.makeScope pkgs.newScope (self: {
       # closure as python can only ever have one version of a dependency
       baseOverlay = self: super:
         let
-          getDep = depName: self.${depName};
           lockPkgs = builtins.listToAttrs (
             builtins.map
               (
-                pkgMeta: rec {
-                  name = moduleName pkgMeta.name;
+                pkgMeta:
+                if builtins.elem pkgMeta.name nixpkgsBuildSystems then {
+                  name = pkgMeta.name;
+                  value = super."${pkgMeta.name}";
+                } else rec {
+                  name = normalizePackageName pkgMeta.name;
                   value = self.mkPoetryDep (
                     pkgMeta // {
                       inherit pwd preferWheels;
@@ -196,17 +194,16 @@ lib.makeScope pkgs.newScope (self: {
                       files = lockFiles.${name};
                       pythonPackages = self;
 
-                      # Packages can be specified with underscores in pyproject.toml; check for
-                      # both possibilities.
-                      sourceSpec = with pyProject.tool.poetry; (
-                        dependencies.${pkgMeta.name} or
-                          dependencies.${underscorify pkgMeta.name} or
-                            dev-dependencies.${pkgMeta.name} or
-                              dev-dependencies.${underscorify pkgMeta.name} or
-                                group.dev.dependencies.${underscorify pkgMeta.name} or # Poetry 1.2.0+
-                                  { }
-
-                      );
+                      sourceSpec =
+                        let
+                          normalizedName = normalizePackageName pkgMeta.name;
+                        in
+                        with pyProject.tool.poetry; (
+                          (normalizePackageSet dependencies).${normalizedName}
+                            or (normalizePackageSet dev-dependencies).${normalizedName}
+                            or (normalizePackageSet group.dev.dependencies).${normalizedName} # Poetry 1.2.0+
+                            or { }
+                        );
                     }
                   );
                 }
@@ -255,7 +252,7 @@ lib.makeScope pkgs.newScope (self: {
               super)
 
             # Null out any filtered packages, we don't want python.pkgs from nixpkgs
-            (self: super: builtins.listToAttrs (builtins.map (x: { name = moduleName x.name; value = null; }) incompatible))
+            (self: super: builtins.listToAttrs (builtins.map (x: { name = normalizePackageName x.name; value = null; }) incompatible))
             # Create poetry2nix layer
             baseOverlay
 
@@ -303,7 +300,7 @@ lib.makeScope pkgs.newScope (self: {
     , groups ? [ "dev" ]
     }:
     let
-      inherit (lib) elem hasAttr;
+      inherit (lib) hasAttr;
 
       pyProject = readTOML pyproject;
 
@@ -395,7 +392,7 @@ lib.makeScope pkgs.newScope (self: {
             py.pkgs.removeGitDependenciesHook
           ];
         } // {
-          pname = moduleName pyProject.tool.poetry.name;
+          pname = normalizePackageName pyProject.tool.poetry.name;
           version = pyProject.tool.poetry.version;
 
           inherit src;
@@ -477,7 +474,7 @@ lib.makeScope pkgs.newScope (self: {
 
     Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function
   */
-  defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib; });
+  defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib poetryLib; });
 
   /*
     Convenience functions for specifying overlays with or without the poerty2nix default overrides

editable.nix

@@ -6,7 +6,7 @@
 , editablePackageSources
 }:
 let
-  name = poetryLib.moduleName pyProject.tool.poetry.name;
+  name = poetryLib.normalizePackageName pyProject.tool.poetry.name;
 
   # Just enough standard PKG-INFO fields for an editable installation
   pkgInfoFields = {

known-build-systems.json

@@ -0,0 +1,12 @@
+[
+    "poetry",
+    "poetry-core",
+    "flit",
+    "flit-core",
+    "pbr",
+    "flitBuildHook",
+    "cython",
+    "hatchling",
+    "setuptools",
+    "setuptools-scm"
+]

lib.nix

@@ -8,12 +8,16 @@ let
     genList (i: if i == idx then value else (builtins.elemAt list i)) (length list)
   );
 
-  # Do some canonicalisation of module names
-  moduleName = name: lib.toLower (lib.replaceStrings [ "_" "." ] [ "-" "-" ] name);
+  # Normalize package names as per PEP 503
+  normalizePackageName = name:
+    let
+      parts = builtins.split "[-_.]+" name;
+      partsWithoutSeparator = builtins.filter (x: builtins.typeOf x == "string") parts;
+    in
+    lib.strings.toLower (lib.strings.concatStringsSep "-" partsWithoutSeparator);
 
-  # For some reason, poetry replaces underscores with dashes in module
-  # names, this has to be reversed sometimes
-  underscorify = name: (lib.replaceStrings [ "-" ] [ "_" ] name);
+  # Normalize an entire attrset of packages
+  normalizePackageSet = lib.attrsets.mapAttrs' (name: value: lib.attrsets.nameValuePair (normalizePackageName name) value);
 
   # Get a full semver pythonVersion from a python derivation
   getPythonVersion = python:
@@ -237,8 +241,8 @@ in
     getBuildSystemPkgs
     satisfiesSemver
     cleanPythonSources
-    moduleName
-    underscorify
+    normalizePackageName
+    normalizePackageSet
     getPythonVersion
     getTargetMachine
     ;

mk-poetry-dep.nix

@@ -26,7 +26,7 @@ pythonPackages.callPackage
     }@args:
     let
       inherit (python) stdenv;
-      inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi moduleName;
+      inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi normalizePackageName;
 
       inherit (import ./pep425.nix {
         inherit lib poetryLib python stdenv;
@@ -88,23 +88,10 @@ pythonPackages.callPackage
             else (builtins.elemAt (lib.strings.splitString "-" name) 2);
         };
 
-      # Prevent infinite recursion
-      skipSetupToolsSCM = [
-        "setuptools_scm"
-        "setuptools-scm"
-        "toml" # Toml is an extra for setuptools-scm
-        "tomli" # tomli is an extra for later versions of setuptools-scm
-        "flit-core"
-        "packaging"
-        "six"
-        "pyparsing"
-        "typing-extensions"
-      ];
-      baseBuildInputs = lib.optional (! lib.elem name skipSetupToolsSCM) pythonPackages.setuptools-scm;
       format = if isDirectory || isGit || isUrl then "pyproject" else fileInfo.format;
     in
     buildPythonPackage {
-      pname = moduleName name;
+      pname = normalizePackageName name;
       version = version;
 
       inherit format;
@@ -124,10 +111,9 @@ pythonPackages.callPackage
       ];
 
       buildInputs = (
-        baseBuildInputs
-        ++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pythonPackages.setuptools
-        ++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg
+        lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg
         ++ lib.optional isDirectory buildSystemPkgs
+        ++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pythonPackages.setuptools
       );
 
       propagatedBuildInputs =
@@ -149,7 +135,7 @@ pythonPackages.callPackage
             );
           depAttrs = lib.attrNames deps;
         in
-        builtins.map (n: pythonPackages.${moduleName n}) depAttrs;
+        builtins.map (n: pythonPackages.${normalizePackageName n}) depAttrs;
 
       meta = {
         broken = ! isCompatible (poetryLib.getPythonVersion python) python-versions;

overrides/default.nix

@@ -1,6 +1,7 @@
 { pkgs ? import <nixpkgs> { }
 , lib ? pkgs.lib
 , stdenv ? pkgs.stdenv
+, poetryLib
 }:
 
 let
@@ -38,6 +39,7 @@ let
       # Flit only works on Python3
       if (attr == "flit-core" || attr == "flit" || attr == "hatchling") && !self.isPy3k then drv
       else if drv == null then null
+      else if drv ? overridePythonAttrs == false then drv
       else
         drv.overridePythonAttrs (
           old:
@@ -57,6 +59,8 @@ let
 
 in
 lib.composeManyExtensions [
+  # normalize all the names
+  (self: super: poetryLib.normalizePackageSet super)
 
   # NixOps
   (self: super:
@@ -81,22 +85,11 @@ lib.composeManyExtensions [
         systems)
       buildSystems)
 
-  # Build systems with conditionals
-  (self: super: {
-
-    platformdirs =
-      if lib.versionAtLeast super.platformdirs.version "2.5.2"
-      then addBuildSystem { inherit self; drv = super.platformdirs; attr = "hatchling"; extraAttrs = [ "hatch-vcs" ]; }
-      else super.platformdirs;
-
-  })
-
   # Build fixes
   (self: super:
     let
       inherit (self.python) stdenv;
       inherit (pkgs.buildPackages) pkg-config;
-      inherit (pkgs) buildPackages;
       pyBuildPackages = self.python.pythonForBuild.pkgs;
 
       selectQt5 = version:
@@ -109,7 +102,7 @@ lib.composeManyExtensions [
 
     {
       automat = super.automat.overridePythonAttrs (
-        old: rec {
+        old: {
           propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.m2r ];
         }
       );
@@ -153,7 +146,7 @@ lib.composeManyExtensions [
       );
 
       argcomplete = super.argcomplete.overridePythonAttrs (
-        old: rec {
+        old: {
           buildInputs = (old.buildInputs or [ ]) ++ [ self.importlib-metadata ];
         }
       );
@@ -165,7 +158,7 @@ lib.composeManyExtensions [
       );
 
       astroid = super.astroid.overridePythonAttrs (
-        old: rec {
+        old: {
           buildInputs = (old.buildInputs or [ ]) ++ [ self.pytest-runner ];
         }
       );
@@ -657,6 +650,11 @@ lib.composeManyExtensions [
         outputs = [ "out" "dev" ];
       });
 
+      gunicorn = super.gunicorn.overridePythonAttrs (old: {
+        # actually needs setuptools as a runtime dependency
+        propagatedBuildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools ];
+      });
+
       h3 = super.h3.overridePythonAttrs (
         old: {
           preBuild = (old.preBuild or "") + ''
@@ -877,7 +875,7 @@ lib.composeManyExtensions [
         else super.jsonschema;
 
       jupyter = super.jupyter.overridePythonAttrs (
-        old: rec {
+        old: {
           # jupyter is a meta-package. Everything relevant comes from the
           # dependencies. It does however have a jupyter.py file that conflicts
           # with jupyter-core so this meta solves this conflict.
@@ -890,7 +888,7 @@ lib.composeManyExtensions [
       });
 
       jupyterlab-widgets = super.jupyterlab-widgets.overridePythonAttrs (
-        old: rec {
+        old: {
           buildInputs = (old.buildInputs or [ ]) ++ [ self.jupyter-packaging ];
         }
       );
@@ -1131,11 +1129,11 @@ lib.composeManyExtensions [
                   excludes = [ "pyproject.toml" ];
                 })
               ];
-              buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools-scm-git-archive ];
+              buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools self.setuptools-scm self.setuptools-scm-git-archive ];
             }
           )) else
           super.molecule.overridePythonAttrs (old: {
-            buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools-scm-git-archive ];
+            buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools self.setuptools-scm self.setuptools-scm-git-archive ];
           });
 
       mpi4py = super.mpi4py.overridePythonAttrs (
@@ -1300,7 +1298,7 @@ lib.composeManyExtensions [
       );
 
       openexr = super.openexr.overridePythonAttrs (
-        old: rec {
+        old: {
           buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.openexr pkgs.ilmbase ];
           NIX_CFLAGS_COMPILE = [ "-I${pkgs.openexr.dev}/include/OpenEXR" "-I${pkgs.ilmbase.dev}/include/OpenEXR" ];
         }
@@ -1385,7 +1383,7 @@ lib.composeManyExtensions [
       });
 
       parsel = super.parsel.overridePythonAttrs (
-        old: rec {
+        old: {
           nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.pytest-runner ];
         }
       );
@@ -1934,7 +1932,7 @@ lib.composeManyExtensions [
       );
 
       rockset = super.rockset.overridePythonAttrs (
-        old: rec {
+        old: {
           postPatch = ''
             cp ./setup_rockset.py ./setup.py
           '';

tests/default.nix

@@ -121,5 +121,6 @@ builtins.removeAttrs
   # Once this is available in 19.09 and unstable we can re-enable the manylinux test
   manylinux = callTest ./manylinux { };
   shapely = callTest ./shapely { };
+  setuptools = callTest ./setuptools { };
 }
   skipTests

tests/extras/poetry.lock

@@ -327,7 +327,7 @@ python-versions = "*"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.6"
-content-hash = "5a83192fb095e50ac54daf8b7d6c63abf824fdd5d87458b93bfe88b0e201c0c6"
+content-hash = "29715f340ec421fd9e3d4ec49f4363e9129714832fb17282c8b8dba9b3833db7"
 
 [metadata.files]
 boto3-stubs = [

tests/extras/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Your Name <you@example.com>"]
 
 [tool.poetry.dependencies]
 python = "^3.6"
-setuptools-scm = { version = "^3.4", extras = ["toml"] }
+setuptools-scm = { version = ">=3.4", extras = ["toml"] }
 
 # Extra with dash in the name should work
 boto3-stubs = {extras = ["cognito-idp"], version = "^1.17.82"}

tests/setuptools/default.nix

@@ -0,0 +1,8 @@
+{ lib, poetry2nix, python3 }:
+
+poetry2nix.mkPoetryApplication {
+  python = python3;
+  pyproject = ./pyproject.toml;
+  poetrylock = ./poetry.lock;
+  src = lib.cleanSource ./.;
+}

tests/setuptools/poetry.lock

@@ -0,0 +1,23 @@
+[[package]]
+name = "setuptools"
+version = "65.4.1"
+description = "Easily download, build, install, upgrade, and uninstall Python packages"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
+testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
+testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
+
+[metadata]
+lock-version = "1.1"
+python-versions = "^3.10"
+content-hash = "bbc2fde99c9aade70aa71c355e2067e25e7d3993814d132cf6a9d8acbb3f251e"
+
+[metadata.files]
+setuptools = [
+    {file = "setuptools-65.4.1-py3-none-any.whl", hash = "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012"},
+    {file = "setuptools-65.4.1.tar.gz", hash = "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"},
+]

tests/setuptools/pyproject.toml

@@ -0,0 +1,13 @@
+[tool.poetry]
+name = "setuptools_test"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+setuptools = "*"
+
+[build-system]
+requires = ["poetry-core>=1.1"]
+build-backend = "poetry.core.masonry.api"

tools/find-build-systems.py

@@ -2,11 +2,13 @@
 from concurrent.futures import ThreadPoolExecutor
 from itertools import chain
 from posix import cpu_count
+import re
 from typing import (
     Dict,
     List,
     Set,
 )
+from pathlib import Path
 import subprocess
 import pynixutil
 import tempfile
@@ -21,16 +23,8 @@ import sys
 
 
 # All known PEP-517 (or otherwise) build systems
-BUILD_SYSTEMS = [
-    "poetry",
-    "poetry-core",
-    "flit",
-    "flit-core",
-    "pbr",
-    "flitBuildHook",
-    "cython",
-    "hatchling",
-]
+with (Path(__file__).parent.parent / "known-build-systems.json").open() as _fd:
+    BUILD_SYSTEMS = json.load(_fd)
 
 
 # Skip these attributes as they have more complex conditions manually
@@ -40,10 +34,16 @@ SKIP_ATTRS = {
     "packaging",
     "poetry",
     "flitBuildHook",
+    "jsonschema",
     "platformdirs",
+    "traitlets",
 }
 
 
+def normalize(name):
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
 def find_known_systems() -> Dict[str, str]:
     """Create a map from attribute to drvPath for known build systems"""
 
@@ -130,15 +130,31 @@ def get_build_systems(known_systems) -> Dict[str, List[str]]:
     return {attr: systems for attr, systems in build_systems.items() if systems}
 
 
+BLOCKLIST = {"poetry", "poetry-core"}
+
+
+def merge_systems(s):
+    simple = {i for i in s if isinstance(i, str)}
+    complex = [i for i in s if isinstance(i, dict)]
+    complex_names = {i["buildSystem"] for i in complex}
+    new_simple = simple - complex_names
+    return complex + sorted(list(new_simple))
+
+
 def merge(prev_content, new_content):
     content = {}
     for attr, systems in chain(prev_content.items(), new_content.items()):
-        s = content.setdefault(attr, set())
+        attr = normalize(attr)
+        s = content.setdefault(attr, [])
         for system in systems:
-            s.add(system)
+            s.append(system)
 
     # Return with sorted data
-    return {attr: sorted(content[attr]) for attr in sorted(content.keys())}
+    return {
+        attr: merge_systems(content[attr])
+        for attr in sorted(content.keys())
+        if attr not in BLOCKLIST
+    }
 
 
 def main():