Merge pull request #94 from RaitoBezarius/raito/fixed-output-for-wheels

Fixed output derivation for wheels fetching

fetch-wheel.sh

@@ -0,0 +1,24 @@
+source $stdenv/setup
+set -euo pipefail
+
+curl="curl            \
+ --location           \
+ --max-redirs 20      \
+ --retry 2            \
+ --disable-epsv       \
+ --cookie-jar cookies \
+ --insecure           \
+ --speed-time 5       \
+ -#                   \
+ --fail               \
+ $curlOpts            \
+ $NIX_CURL_FLAGS"
+
+echo "Trying to fetch wheel with predicted URL: $predictedURL"
+
+$curl $predictedURL --output $out && exit 0
+
+echo "Predicted URL '$predictedURL' failed, querying pypi.org"
+$curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$file\") | .url" > url
+url=$(cat url)
+$curl -k $url --output $out

lib.nix

@@ -82,6 +82,57 @@ let
     else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; }
     else { pkg = [ ]; str = null; };
 
+  # Predict URL from the PyPI index.
+  # Args:
+  #   pname: package name
+  #   file: filename including extension
+  #   hash: SRI hash
+  #   kind: Language implementation and version tag
+  predictURLFromPypi = lib.makeOverridable
+    (
+      { pname, file, hash, kind }:
+      "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}"
+    );
+
+
+  # Fetch the wheels from the PyPI index.
+  # We need to first get the proper URL to the wheel.
+  # Args:
+  #   pname: package name
+  #   file: filename including extension
+  #   hash: SRI hash
+  #   kind: Language implementation and version tag
+  fetchWheelFromPypi = lib.makeOverridable
+    (
+      { pname, file, hash, kind, curlOpts ? "" }:
+      let
+        version = builtins.elemAt (builtins.split "-" file) 2;
+      in
+      (pkgs.stdenvNoCC.mkDerivation {
+        name = file;
+        nativeBuildInputs = [
+          pkgs.curl
+          pkgs.jq
+        ];
+        isWheel = true;
+        system = "builtin";
+
+        preferLocalBuild = true;
+        impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
+          "NIX_CURL_FLAGS"
+        ];
+
+        predictedURL = predictURLFromPypi { inherit pname file hash kind; };
+        inherit pname file version curlOpts;
+
+        builder = ./fetch-wheel.sh;
+
+        outputHashMode = "flat";
+        outputHashAlgo = "sha256";
+        outputHash = hash;
+      })
+    );
+
   # Fetch the artifacts from the PyPI index. Since we get all
   # info we need from the lock file we don't use nixpkgs' fetchPyPi
   # as it modifies casing while not providing anything we don't already
@@ -95,8 +146,10 @@ let
   fetchFromPypi = lib.makeOverridable
     (
       { pname, file, hash, kind }:
+      if lib.strings.hasSuffix "whl" file then fetchWheelFromPypi { inherit pname file hash kind; }
+      else
         pkgs.fetchurl {
-        url = "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}";
+          url = predictURLFromPypi { inherit pname file hash kind; };
           inherit hash;
         }
     );
@@ -149,6 +202,7 @@ in
 {
   inherit
     fetchFromPypi
+    fetchWheelFromPypi
     getManyLinuxDeps
     isCompatible
     readTOML

overrides.nix

@@ -145,14 +145,15 @@ self: super:
 
   h5py = super.h5py.overridePythonAttrs
     (
-      old: rec {
+      old:
+      if old.format != "wheel" then rec {
         nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ];
         buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkgconfig self.cython ];
         configure_flags = "--hdf5=${pkgs.hdf5}";
         postConfigure = ''
           ${self.python.executable} setup.py configure ${configure_flags}
         '';
-      }
+      } else old
     );
 
   horovod = super.horovod.overridePythonAttrs
@@ -700,7 +701,8 @@ self: super:
 
   scipy = super.scipy.overridePythonAttrs
     (
-      old: {
+      old:
+      if old.format != "wheel" then {
         nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ];
         propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.pybind11 ];
         setupPyBuildFlags = [ "--fcompiler='gnu95'" ];
@@ -713,7 +715,7 @@ self: super:
         preBuild = ''
           ln -s ${self.numpy.cfg} site.cfg
         '';
-      }
+      } else old
     );
 
   scikit-learn = super.scikit-learn.overridePythonAttrs
@@ -805,7 +807,8 @@ self: super:
     }
   ).wheel.overridePythonAttrs
     (
-      _: {
+      old:
+      if old.format == "other" then old else {
         inherit (super.wheel) pname name version src;
       }
     );

tests/prefer-wheel/default.nix

@@ -15,6 +15,6 @@ let
         }
       );
   };
-  url = lib.elemAt drv.passthru.python.pkgs.maturin.src.urls 0;
+  isWheelAttr = drv.passthru.python.pkgs.maturin.src.isWheel or false;
 in
-  assert lib.hasSuffix "whl" url; drv
+  assert isWheelAttr; drv

tests/prefer-wheels/default.nix

@@ -1,11 +1,9 @@
 { lib, poetry2nix, python3, runCommand }:
 let
-  app = poetry2nix.mkPoetryApplication {
+  py = poetry2nix.mkPoetryPackages {
     projectDir = ./.;
     preferWheels = true;
   };
-  url = lib.elemAt app.passthru.python.pkgs.tensorflow.src.urls 0;
+  isWheelAttr = py.python.pkgs.tensorflow.src.isWheel or false;
 in
-  assert lib.hasSuffix "whl" url; runCommand "prefer-wheels" { } ''
+  assert isWheelAttr; (py.python.withPackages (_: py.poetryPackages)).override (args: { ignoreCollisions = true; })
-    touch $out
-  ''

tests/prefer-wheels/poetry.lock

@@ -362,7 +362,7 @@ description = "TensorFlow Estimator."
 name = "tensorflow-estimator"
 optional = false
 python-versions = "*"
-version = "2.2.0rc0"
+version = "2.1.0rc0"
 
 [[package]]
 category = "main"
@@ -418,7 +418,7 @@ python-versions = "*"
 version = "1.12.1"
 
 [metadata]
-content-hash = "f1c052ba8df831358ef7630291473df2e0310dc128e9e4e204d746a86cae6b99"
+content-hash = "82b72e0ca256c371b2b647204ea22345c297fb6121831320dbc7a85254bdf478"
 python-versions = "^3.6"
 
 [metadata.files]
@@ -676,7 +676,7 @@ tensorflow = [
     {file = "tensorflow-2.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7bad8ea686a1f33d9dac13eb578c4597346789d4f826980c8bbcfbd08e7dc921"},
 ]
 tensorflow-estimator = [
-    {file = "tensorflow_estimator-2.2.0rc0-py2.py3-none-any.whl", hash = "sha256:c24aba63b33de5db089e66585399e1abefad75038c4aacca8d8d1328ab3e8282"},
+    {file = "tensorflow_estimator-2.1.0rc0-py2.py3-none-any.whl", hash = "sha256:7d835eeb7eaff4d3c3d640be9d9296e9a9213b6172cb4a95317817d180e682c7"},
 ]
 termcolor = [
     {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"},

tests/prefer-wheels/pyproject.toml

@@ -7,6 +7,7 @@ authors = ["Your Name <you@example.com>"]
 [tool.poetry.dependencies]
 python = "^3.6"
 tensorflow = "^2.1.0"
+tensorflow-estimator = "2.1.0rc0"
 
 [tool.poetry.dev-dependencies]