diff --git a/fetch-wheel.sh b/fetch-wheel.sh new file mode 100644 index 0000000..97f54b2 --- /dev/null +++ b/fetch-wheel.sh @@ -0,0 +1,24 @@ +source $stdenv/setup +set -euo pipefail + +curl="curl \ + --location \ + --max-redirs 20 \ + --retry 2 \ + --disable-epsv \ + --cookie-jar cookies \ + --insecure \ + --speed-time 5 \ + -# \ + --fail \ + $curlOpts \ + $NIX_CURL_FLAGS" + +echo "Trying to fetch wheel with predicted URL: $predictedURL" + +$curl $predictedURL --output $out && exit 0 + +echo "Predicted URL '$predictedURL' failed, querying pypi.org" +$curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$file\") | .url" > url +url=$(cat url) +$curl -k $url --output $out diff --git a/lib.nix b/lib.nix index 01a15e0..ed47837 100644 --- a/lib.nix +++ b/lib.nix @@ -82,6 +82,57 @@ let else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; } else { pkg = [ ]; str = null; }; + # Predict URL from the PyPI index. + # Args: + # pname: package name + # file: filename including extension + # hash: SRI hash + # kind: Language implementation and version tag + predictURLFromPypi = lib.makeOverridable + ( + { pname, file, hash, kind }: + "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}" + ); + + + # Fetch the wheels from the PyPI index. + # We need to first get the proper URL to the wheel. + # Args: + # pname: package name + # file: filename including extension + # hash: SRI hash + # kind: Language implementation and version tag + fetchWheelFromPypi = lib.makeOverridable + ( + { pname, file, hash, kind, curlOpts ? "" }: + let + version = builtins.elemAt (builtins.split "-" file) 2; + in + (pkgs.stdenvNoCC.mkDerivation { + name = file; + nativeBuildInputs = [ + pkgs.curl + pkgs.jq + ]; + isWheel = true; + system = "builtin"; + + preferLocalBuild = true; + impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ + "NIX_CURL_FLAGS" + ]; + + predictedURL = predictURLFromPypi { inherit pname file hash kind; }; + inherit pname file version curlOpts; + + builder = ./fetch-wheel.sh; + + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = hash; + }) + ); + # Fetch the artifacts from the PyPI index. Since we get all # info we need from the lock file we don't use nixpkgs' fetchPyPi # as it modifies casing while not providing anything we don't already @@ -95,10 +146,12 @@ let fetchFromPypi = lib.makeOverridable ( { pname, file, hash, kind }: - pkgs.fetchurl { - url = "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}"; - inherit hash; - } + if lib.strings.hasSuffix "whl" file then fetchWheelFromPypi { inherit pname file hash kind; } + else + pkgs.fetchurl { + url = predictURLFromPypi { inherit pname file hash kind; }; + inherit hash; + } ); getBuildSystemPkgs = { pythonPackages @@ -149,6 +202,7 @@ in { inherit fetchFromPypi + fetchWheelFromPypi getManyLinuxDeps isCompatible readTOML diff --git a/overrides.nix b/overrides.nix index d8d35e3..37c5d18 100644 --- a/overrides.nix +++ b/overrides.nix @@ -145,14 +145,15 @@ self: super: h5py = super.h5py.overridePythonAttrs ( - old: rec { + old: + if old.format != "wheel" then rec { nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkgconfig self.cython ]; configure_flags = "--hdf5=${pkgs.hdf5}"; postConfigure = '' ${self.python.executable} setup.py configure ${configure_flags} ''; - } + } else old ); horovod = super.horovod.overridePythonAttrs @@ -700,7 +701,8 @@ self: super: scipy = super.scipy.overridePythonAttrs ( - old: { + old: + if old.format != "wheel" then { nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ]; propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.pybind11 ]; setupPyBuildFlags = [ "--fcompiler='gnu95'" ]; @@ -713,7 +715,7 @@ self: super: preBuild = '' ln -s ${self.numpy.cfg} site.cfg ''; - } + } else old ); scikit-learn = super.scikit-learn.overridePythonAttrs @@ -805,7 +807,8 @@ self: super: } ).wheel.overridePythonAttrs ( - _: { + old: + if old.format == "other" then old else { inherit (super.wheel) pname name version src; } ); diff --git a/tests/prefer-wheel/default.nix b/tests/prefer-wheel/default.nix index 6ff6e36..4be0faa 100644 --- a/tests/prefer-wheel/default.nix +++ b/tests/prefer-wheel/default.nix @@ -15,6 +15,6 @@ let } ); }; - url = lib.elemAt drv.passthru.python.pkgs.maturin.src.urls 0; + isWheelAttr = drv.passthru.python.pkgs.maturin.src.isWheel or false; in - assert lib.hasSuffix "whl" url; drv + assert isWheelAttr; drv diff --git a/tests/prefer-wheels/default.nix b/tests/prefer-wheels/default.nix index 3eca0b3..8639609 100644 --- a/tests/prefer-wheels/default.nix +++ b/tests/prefer-wheels/default.nix @@ -1,11 +1,9 @@ { lib, poetry2nix, python3, runCommand }: let - app = poetry2nix.mkPoetryApplication { + py = poetry2nix.mkPoetryPackages { projectDir = ./.; preferWheels = true; }; - url = lib.elemAt app.passthru.python.pkgs.tensorflow.src.urls 0; + isWheelAttr = py.python.pkgs.tensorflow.src.isWheel or false; in - assert lib.hasSuffix "whl" url; runCommand "prefer-wheels" { } '' - touch $out - '' + assert isWheelAttr; (py.python.withPackages (_: py.poetryPackages)).override (args: { ignoreCollisions = true; }) diff --git a/tests/prefer-wheels/poetry.lock b/tests/prefer-wheels/poetry.lock index a349631..b897134 100644 --- a/tests/prefer-wheels/poetry.lock +++ b/tests/prefer-wheels/poetry.lock @@ -362,7 +362,7 @@ description = "TensorFlow Estimator." name = "tensorflow-estimator" optional = false python-versions = "*" -version = "2.2.0rc0" +version = "2.1.0rc0" [[package]] category = "main" @@ -418,7 +418,7 @@ python-versions = "*" version = "1.12.1" [metadata] -content-hash = "f1c052ba8df831358ef7630291473df2e0310dc128e9e4e204d746a86cae6b99" +content-hash = "82b72e0ca256c371b2b647204ea22345c297fb6121831320dbc7a85254bdf478" python-versions = "^3.6" [metadata.files] @@ -676,7 +676,7 @@ tensorflow = [ {file = "tensorflow-2.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7bad8ea686a1f33d9dac13eb578c4597346789d4f826980c8bbcfbd08e7dc921"}, ] tensorflow-estimator = [ - {file = "tensorflow_estimator-2.2.0rc0-py2.py3-none-any.whl", hash = "sha256:c24aba63b33de5db089e66585399e1abefad75038c4aacca8d8d1328ab3e8282"}, + {file = "tensorflow_estimator-2.1.0rc0-py2.py3-none-any.whl", hash = "sha256:7d835eeb7eaff4d3c3d640be9d9296e9a9213b6172cb4a95317817d180e682c7"}, ] termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, diff --git a/tests/prefer-wheels/pyproject.toml b/tests/prefer-wheels/pyproject.toml index 48660dc..0ecb8de 100644 --- a/tests/prefer-wheels/pyproject.toml +++ b/tests/prefer-wheels/pyproject.toml @@ -7,6 +7,7 @@ authors = ["Your Name "] [tool.poetry.dependencies] python = "^3.6" tensorflow = "^2.1.0" +tensorflow-estimator = "2.1.0rc0" [tool.poetry.dev-dependencies]