From cd693810c3644ad3392d847f8af7d999689ca7bd Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Thu, 13 May 2021 19:16:58 +0200 Subject: [PATCH 1/3] Always fallback to the Pypi API when fetching sources & wheels Previously we randomly encountered issues where the Pypi mirror URLs wouldn't be correct with packages whos first character is frequently written in upper case (e.g. SqlAlchemy, MarkupSafe, ...). The Pypi mirrors are aparently not consistent in the naming of the file locations which lead to random errors after package version bumps. By falling back to the API lookup (that we already did for wheels) we can workaround that situation. --- fetch-wheel.sh => fetch-from-pypi.sh | 2 +- lib.nix | 43 +++++++++------------------- mk-poetry-dep.nix | 1 + 3 files changed, 16 insertions(+), 30 deletions(-) rename fetch-wheel.sh => fetch-from-pypi.sh (90%) diff --git a/fetch-wheel.sh b/fetch-from-pypi.sh similarity index 90% rename from fetch-wheel.sh rename to fetch-from-pypi.sh index 97f54b2..398360f 100644 --- a/fetch-wheel.sh +++ b/fetch-from-pypi.sh @@ -14,7 +14,7 @@ curl="curl \ $curlOpts \ $NIX_CURL_FLAGS" -echo "Trying to fetch wheel with predicted URL: $predictedURL" +echo "Trying to fetch with predicted URL: $predictedURL" $curl $predictedURL --output $out && exit 0 diff --git a/lib.nix b/lib.nix index 6af37b3..c6366f4 100644 --- a/lib.nix +++ b/lib.nix @@ -93,17 +93,19 @@ let ); - # Fetch the wheels from the PyPI index. - # We need to first get the proper URL to the wheel. + # Fetch from the PyPI index. + # At first we try to fetch the predicated URL but if that fails we + # will use the Pypi API to determine the correct URL. # Args: # pname: package name # file: filename including extension + # version: the version string of the dependency # hash: SRI hash # kind: Language implementation and version tag - fetchWheelFromPypi = lib.makeOverridable ( - { pname, file, hash, kind, curlOpts ? "" }: + fetchFromPypi = lib.makeOverridable ( + { pname, file, version, hash, kind, curlOpts ? "" }: let - version = builtins.elemAt (builtins.split "-" file) 2; + predictedURL = predictURLFromPypi { inherit pname file hash kind; }; in (pkgs.stdenvNoCC.mkDerivation { name = file; @@ -111,7 +113,7 @@ let pkgs.curl pkgs.jq ]; - isWheel = true; + isWheel = lib.strings.hasSuffix "whl" file; system = "builtin"; preferLocalBuild = true; @@ -119,36 +121,20 @@ let "NIX_CURL_FLAGS" ]; - predictedURL = predictURLFromPypi { inherit pname file hash kind; }; - inherit pname file version curlOpts; + inherit pname file version curlOpts predictedURL; - builder = ./fetch-wheel.sh; + builder = ./fetch-from-pypi.sh; outputHashMode = "flat"; outputHashAlgo = "sha256"; outputHash = hash; + + passthru = { + urls = [ predictedURL ]; # retain compatibility with nixpkgs' fetchurl + }; }) ); - # Fetch the artifacts from the PyPI index. Since we get all - # info we need from the lock file we don't use nixpkgs' fetchPyPi - # as it modifies casing while not providing anything we don't already - # have. - # - # Args: - # pname: package name - # file: filename including extension - # hash: SRI hash - # kind: Language implementation and version tag https://www.python.org/dev/peps/pep-0427/#file-name-convention - fetchFromPypi = lib.makeOverridable ( - { pname, file, hash, kind }: - if lib.strings.hasSuffix "whl" file then fetchWheelFromPypi { inherit pname file hash kind; } - else - pkgs.fetchurl { - url = predictURLFromPypi { inherit pname file hash kind; }; - inherit hash; - } - ); getBuildSystemPkgs = { pythonPackages , pyProject @@ -215,7 +201,6 @@ in { inherit fetchFromPypi - fetchWheelFromPypi getManyLinuxDeps isCompatible readTOML diff --git a/mk-poetry-dep.nix b/mk-poetry-dep.nix index bb7b4e3..71bee4d 100644 --- a/mk-poetry-dep.nix +++ b/mk-poetry-dep.nix @@ -175,6 +175,7 @@ pythonPackages.callPackage fetchFromPypi { pname = name; inherit (fileInfo) file hash kind; + inherit version; }; } ) From a2854eaae0b2b9306d892228a162263148e80a8e Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Thu, 13 May 2021 20:35:35 +0200 Subject: [PATCH 2/3] Expand the -# flag in fetch-from-pypi.sh --- fetch-from-pypi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch-from-pypi.sh b/fetch-from-pypi.sh index 398360f..e56dee6 100644 --- a/fetch-from-pypi.sh +++ b/fetch-from-pypi.sh @@ -9,7 +9,7 @@ curl="curl \ --cookie-jar cookies \ --insecure \ --speed-time 5 \ - -# \ + --progress-bar \ --fail \ $curlOpts \ $NIX_CURL_FLAGS" From 7113a96c35e066e44cc7ecb4c4440560dd2c4285 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Thu, 13 May 2021 21:46:40 +0200 Subject: [PATCH 3/3] Add MarkupSafe2 test case This test case is what initially motivated the move towards Pypi API based fallback fetching. --- tests/default.nix | 1 + tests/markupsafe2/default.nix | 7 +++++ tests/markupsafe2/poetry.lock | 50 ++++++++++++++++++++++++++++++++ tests/markupsafe2/pyproject.toml | 16 ++++++++++ 4 files changed, 74 insertions(+) create mode 100644 tests/markupsafe2/default.nix create mode 100644 tests/markupsafe2/poetry.lock create mode 100644 tests/markupsafe2/pyproject.toml diff --git a/tests/default.nix b/tests/default.nix index 74ebd43..d067cc9 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -50,6 +50,7 @@ builtins.removeAttrs editable-egg = callTest ./editable-egg { }; ansible-molecule = callTest ./ansible-molecule { }; mk-poetry-packages = callTest ./mk-poetry-packages { }; + markupsafe2 = callTest ./markupsafe2 { }; # Test building poetry inherit poetry; diff --git a/tests/markupsafe2/default.nix b/tests/markupsafe2/default.nix new file mode 100644 index 0000000..5bc41cd --- /dev/null +++ b/tests/markupsafe2/default.nix @@ -0,0 +1,7 @@ +{ lib, poetry2nix, python3 }: + +poetry2nix.mkPoetryEnv { + python = python3; + pyproject = ./pyproject.toml; + poetrylock = ./poetry.lock; +} diff --git a/tests/markupsafe2/poetry.lock b/tests/markupsafe2/poetry.lock new file mode 100644 index 0000000..c53af5f --- /dev/null +++ b/tests/markupsafe2/poetry.lock @@ -0,0 +1,50 @@ +[[package]] +name = "markupsafe" +version = "2.0.0" +description = "Safely add untrusted strings to HTML/XML markup." +category = "main" +optional = false +python-versions = ">=3.6" + +[metadata] +lock-version = "1.1" +python-versions = "^3.6" +content-hash = "4cb03b469367c7275485e67a46d6d3785e2ca3174756575962dd95f17f8fddae" + +[metadata.files] +markupsafe = [ + {file = "MarkupSafe-2.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:2efaeb1baff547063bad2b2893a8f5e9c459c4624e1a96644bbba08910ae34e0"}, + {file = "MarkupSafe-2.0.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:441ce2a8c17683d97e06447fcbccbdb057cbf587c78eb75ae43ea7858042fe2c"}, + {file = "MarkupSafe-2.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:45535241baa0fc0ba2a43961a1ac7562ca3257f46c4c3e9c0de38b722be41bd1"}, + {file = "MarkupSafe-2.0.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:90053234a6479738fd40d155268af631c7fca33365f964f2208867da1349294b"}, + {file = "MarkupSafe-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:3b54a9c68995ef4164567e2cd1a5e16db5dac30b2a50c39c82db8d4afaf14f63"}, + {file = "MarkupSafe-2.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:f58b5ba13a5689ca8317b98439fccfbcc673acaaf8241c1869ceea40f5d585bf"}, + {file = "MarkupSafe-2.0.0-cp36-cp36m-win32.whl", hash = "sha256:a00dce2d96587651ef4fa192c17e039e8cfab63087c67e7d263a5533c7dad715"}, + {file = "MarkupSafe-2.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:007dc055dbce5b1104876acee177dbfd18757e19d562cd440182e1f492e96b95"}, + {file = "MarkupSafe-2.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a08cd07d3c3c17cd33d9e66ea9dee8f8fc1c48e2d11bd88fd2dc515a602c709b"}, + {file = "MarkupSafe-2.0.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:3c352ff634e289061711608f5e474ec38dbaa21e3e168820d53d5f4015e5b91b"}, + {file = "MarkupSafe-2.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:32200f562daaab472921a11cbb63780f1654552ae49518196fc361ed8e12e901"}, + {file = "MarkupSafe-2.0.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:fef86115fdad7ae774720d7103aa776144cf9b66673b4afa9bcaa7af990ed07b"}, + {file = "MarkupSafe-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:e79212d09fc0e224d20b43ad44bb0a0a3416d1e04cf6b45fed265114a5d43d20"}, + {file = "MarkupSafe-2.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:79b2ae94fa991be023832e6bcc00f41dbc8e5fe9d997a02db965831402551730"}, + {file = "MarkupSafe-2.0.0-cp37-cp37m-win32.whl", hash = "sha256:3261fae28155e5c8634dd7710635fe540a05b58f160cef7713c7700cb9980e66"}, + {file = "MarkupSafe-2.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e4570d16f88c7f3032ed909dc9e905a17da14a1c4cfd92608e3fda4cb1208bbd"}, + {file = "MarkupSafe-2.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8f806bfd0f218477d7c46a11d3e52dc7f5fdfaa981b18202b7dc84bbc287463b"}, + {file = "MarkupSafe-2.0.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e77e4b983e2441aff0c0d07ee711110c106b625f440292dfe02a2f60c8218bd6"}, + {file = "MarkupSafe-2.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:031bf79a27d1c42f69c276d6221172417b47cb4b31cdc73d362a9bf5a1889b9f"}, + {file = "MarkupSafe-2.0.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:83cf0228b2f694dcdba1374d5312f2277269d798e65f40344964f642935feac1"}, + {file = "MarkupSafe-2.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:4cc563836f13c57f1473bc02d1e01fc37bab70ad4ee6be297d58c1d66bc819bf"}, + {file = "MarkupSafe-2.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:d00a669e4a5bec3ee6dbeeeedd82a405ced19f8aeefb109a012ea88a45afff96"}, + {file = "MarkupSafe-2.0.0-cp38-cp38-win32.whl", hash = "sha256:161d575fa49395860b75da5135162481768b11208490d5a2143ae6785123e77d"}, + {file = "MarkupSafe-2.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:58bc9fce3e1557d463ef5cee05391a05745fd95ed660f23c1742c711712c0abb"}, + {file = "MarkupSafe-2.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3fb47f97f1d338b943126e90b79cad50d4fcfa0b80637b5a9f468941dbbd9ce5"}, + {file = "MarkupSafe-2.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dab0c685f21f4a6c95bfc2afd1e7eae0033b403dd3d8c1b6d13a652ada75b348"}, + {file = "MarkupSafe-2.0.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:664832fb88b8162268928df233f4b12a144a0c78b01d38b81bdcf0fc96668ecb"}, + {file = "MarkupSafe-2.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:df561f65049ed3556e5b52541669310e88713fdae2934845ec3606f283337958"}, + {file = "MarkupSafe-2.0.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:24bbc3507fb6dfff663af7900a631f2aca90d5a445f272db5fc84999fa5718bc"}, + {file = "MarkupSafe-2.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:87de598edfa2230ff274c4de7fcf24c73ffd96208c8e1912d5d0fee459767d75"}, + {file = "MarkupSafe-2.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a19d39b02a24d3082856a5b06490b714a9d4179321225bbf22809ff1e1887cc8"}, + {file = "MarkupSafe-2.0.0-cp39-cp39-win32.whl", hash = "sha256:4aca81a687975b35e3e80bcf9aa93fe10cd57fac37bf18b2314c186095f57e05"}, + {file = "MarkupSafe-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:70820a1c96311e02449591cbdf5cd1c6a34d5194d5b55094ab725364375c9eb2"}, + {file = "MarkupSafe-2.0.0.tar.gz", hash = "sha256:4fae0677f712ee090721d8b17f412f1cbceefbf0dc180fe91bab3232f38b4527"}, +] diff --git a/tests/markupsafe2/pyproject.toml b/tests/markupsafe2/pyproject.toml new file mode 100644 index 0000000..68b3955 --- /dev/null +++ b/tests/markupsafe2/pyproject.toml @@ -0,0 +1,16 @@ +[tool.poetry] +name = "markupsafe2" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.6" +MarkupSafe = "^2.0" + + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api"