Merge pull request #577 from SemMulder/feature/pep600

Add PEP 600 (new manylinux) support

lib.nix

@@ -79,6 +79,7 @@ let
     if lib.strings.hasInfix "manylinux1" f then { pkg = [ ml.manylinux1 ]; str = "1"; }
     else if lib.strings.hasInfix "manylinux2010" f then { pkg = [ ml.manylinux2010 ]; str = "2010"; }
     else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; }
+    else if lib.strings.hasInfix "manylinux_" f then { pkg = [ ml.manylinux2014 ]; str = "pep600"; }
     else { pkg = [ ]; str = null; };
 
   # Predict URL from the PyPI index.

overrides/default.nix

@@ -17,9 +17,14 @@ let
       if (attr == "flit-core" || attr == "flit") && !self.isPy3k then drv
       else
         drv.overridePythonAttrs (
-          old: {
-            nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.${attr} ];
-          }
+          old:
+          # We do not need the build system for wheels.
+          if old ? format && old.format == "wheel" then
+            { }
+          else
+            {
+              nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.${attr} ];
+            }
         )
     );
 
@@ -734,6 +739,10 @@ lib.composeManyExtensions [
         }
       );
 
+      jupyter-packaging = super.jupyter-packaging.overridePythonAttrs (old: {
+        propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.wheel ];
+      });
+
       jupyterlab-widgets = super.jupyterlab-widgets.overridePythonAttrs (
         old: rec {
           buildInputs = (old.buildInputs or [ ]) ++ [ self.jupyter-packaging ];
@@ -1084,6 +1093,14 @@ lib.composeManyExtensions [
         }
       );
 
+      open3d = super.open3d.overridePythonAttrs (old: {
+        buildInputs = (old.buildInputs or [ ]) ++ (with pkgs; [
+          udev
+        ]);
+        # TODO(Sem Mulder): Add overridable flags for CUDA/PyTorch/Tensorflow support.
+        autoPatchelfIgnoreMissingDeps = true;
+      });
+
       opencv-python = super.opencv-python.overridePythonAttrs (
         old: {
           nativeBuildInputs = [ pkgs.cmake ] ++ old.nativeBuildInputs;

pep425.nix

@@ -1,6 +1,6 @@
 { lib, stdenv, poetryLib, python, isLinux ? stdenv.isLinux }:
 let
-  inherit (lib.strings) hasSuffix hasInfix splitString removeSuffix;
+  inherit (lib.strings) escapeRegex hasPrefix hasSuffix hasInfix splitString removePrefix removeSuffix;
   targetMachine = poetryLib.getTargetMachine stdenv;
 
   # The 'cpxy" as determined by `python.version`
@@ -52,10 +52,10 @@ let
   # x     = "cpXX" | "py2" | "py3" | "py2.py3"
   isPyVersionCompatible = pyver: x:
     let
-      normalize = y: ''cp${lib.strings.removePrefix "cp" (lib.strings.removePrefix "py" y)}'';
-      isCompat = p: x: lib.strings.hasPrefix (normalize x) p;
+      normalize = y: ''cp${removePrefix "cp" (removePrefix "py" y)}'';
+      isCompat = p: x: hasPrefix (normalize x) p;
     in
-    lib.lists.any (isCompat pyver) (lib.strings.splitString "." x);
+    lib.lists.any (isCompat pyver) (splitString "." x);
 
   #
   # Selects the best matching wheel file from a list of files
@@ -63,7 +63,7 @@ let
   selectWheel = files:
     let
       filesWithoutSources = (builtins.filter (x: hasSuffix ".whl" x.file) files);
-      isPyAbiCompatible = pyabi: x: x == "none" || lib.hasPrefix pyabi x || lib.hasPrefix x pyabi || (
+      isPyAbiCompatible = pyabi: x: x == "none" || hasPrefix pyabi x || hasPrefix x pyabi || (
         # The CPython stable ABI is abi3 as in the shared library suffix.
         python.passthru.implementation == "cpython" &&
           builtins.elemAt (lib.splitString "." python.version) 0 == "3" &&
@@ -75,32 +75,30 @@ let
         then
           if targetMachine != null
           then
-            (
-              x: x.platform == "any" || lib.lists.any (e: hasInfix e x.platform) [
-                "manylinux1_${targetMachine}"
-                "manylinux2010_${targetMachine}"
-                "manylinux2014_${targetMachine}"
-              ]
+          # See PEP 600 for details.
+            (p:
+              builtins.match "any|manylinux(1|2010|2014)_${escapeRegex targetMachine}|manylinux_[0-9]+_[0-9]+_${escapeRegex targetMachine}" p != null
             )
           else
-            (x: x.platform == "any")
+            (p: p == "any")
         else
           if stdenv.isDarwin
           then
             if stdenv.targetPlatform.isAarch64
-            then (x: x.platform == "any" || (hasInfix "macosx" x.platform && lib.lists.any (e: hasSuffix e x.platform) [ "arm64" "aarch64" ]))
-            else (x: x.platform == "any" || (hasInfix "macosx" x.platform && hasSuffix "x86_64" x.platform))
-          else (x: x.platform == "any");
+            then (p: p == "any" || (hasInfix "macosx" p && lib.lists.any (e: hasSuffix e p) [ "arm64" "aarch64" ]))
+            else (p: p == "any" || (hasInfix "macosx" p && hasSuffix "x86_64" p))
+          else (p: p == "any");
+      withPlatforms = x: lib.lists.any withPlatform (splitString "." x.platform);
       filterWheel = x:
         let
           f = toWheelAttrs x.file;
         in
-        (withPython pythonTag abiTag f) && (withPlatform f);
+        (withPython pythonTag abiTag f) && (withPlatforms f);
       filtered = builtins.filter filterWheel filesWithoutSources;
       choose = files:
         let
           osxMatches = [ "12_0" "11_0" "10_12" "10_11" "10_10" "10_9" "10_8" "10_7" "any" ];
-          linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "any" ];
+          linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "manylinux_" "any" ];
           chooseLinux = x: lib.take 1 (findBestMatches linuxMatches x);
           chooseOSX = x: lib.take 1 (findBestMatches osxMatches x);
         in

tests/default.nix

@@ -31,6 +31,7 @@ builtins.removeAttrs
   common-pkgs-1 = callTest ./common-pkgs-1 { };
   common-pkgs-2 = callTest ./common-pkgs-2 { };
   pep425 = pkgs.callPackage ./pep425 { inherit pep425; inherit pep425OSX; inherit pep425Python37; };
+  pep600 = skipOSX (callTest ./pep600 { });
   env = callTest ./env { };
   pytest-randomly = callTest ./pytest-randomly { };
   file-src-deps = callTest ./file-src-deps { };

tests/pep600/default.nix

@@ -0,0 +1,19 @@
+{ lib, poetry2nix, python3, runCommand }:
+let
+  env = poetry2nix.mkPoetryEnv {
+    python = python3;
+    preferWheels = true;
+    pyproject = ./pyproject.toml;
+    poetrylock = ./poetry.lock;
+    overrides = poetry2nix.overrides.withDefaults (self: super: {
+      threadpoolctl = super.threadpoolctl.overridePythonAttrs (old: {
+        format = "wheel";
+      });
+    });
+  };
+in
+runCommand "pep600-test"
+{ } ''
+  ${env}/bin/python -c 'import open3d; print(open3d.__version__)'
+  touch $out
+''

tests/pep600/pyproject.toml

@@ -0,0 +1,16 @@
+[tool.poetry]
+name = "test-pep600"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+open3d = "=0.15.2"
+jupyter-packaging = "<0.12.0"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"