Add support to the legacy Pypi API

Some private repositories (such as Devpi) expose the legacy Pypi
API (https://warehouse.pypa.io/api-reference/legacy.html).

This commit adds a dedicated fetcher which basically queries this API
to get the URL pointing to the actual file.

Since Pypi still exposes this API, it has been possible to write a
test that uses this legacy API.

Fixes https://github.com/nix-community/poetry2nix/issues/277.

fetch_from_legacy.py

@@ -0,0 +1,75 @@
+# Some repositories (such as Devpi) expose the Pypi legacy API
+# (https://warehouse.pypa.io/api-reference/legacy.html).
+#
+# Note it is not possible to use pip
+# https://discuss.python.org/t/pip-download-just-the-source-packages-no-building-no-metadata-etc/4651/12
+
+import sys
+from urllib.parse import urlparse
+from html.parser import HTMLParser
+import urllib.request
+import shutil
+import ssl
+import os
+
+
+# Parse the legacy index page to extract the href and package names
+class Pep503(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.sources = {}
+        self.url = None
+        self.name = None
+
+    def handle_data(self, data):
+        if self.url is not None:
+            self.name = data
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "a":
+            for name, value in attrs:
+                if name == "href":
+                    self.url = value
+
+    def handle_endtag(self, tag):
+        if self.url is not None:
+            self.sources[self.name] = self.url
+        self.url = None
+
+
+url = sys.argv[1]
+package_name = sys.argv[2]
+index_url = url + "/" + package_name
+package_filename = sys.argv[3]
+
+ssl_context = ssl.create_default_context(
+    cafile=os.environ.get("SSL_CERT_FILE"))
+
+print("Reading index %s" % index_url)
+
+response = urllib.request.urlopen(
+    index_url,
+    context=ssl_context)
+index = response.read()
+
+parser = Pep503()
+parser.feed(str(index))
+if package_filename not in parser.sources:
+    print("The file %s has not be found in the index %s" % (
+        package_filename, index_url))
+    exit(1)
+
+package_file = open(package_filename, "wb")
+# Sometimes the href is a relative path
+if urlparse(parser.sources[package_filename]).netloc == '':
+    package_url = index_url + "/" + parser.sources[package_filename]
+else:
+    package_url = parser.sources[package_filename]
+print("Downloading %s" % package_url)
+
+response = urllib.request.urlopen(
+    package_url,
+    context=ssl_context)
+
+with response as r:
+    shutil.copyfileobj(r, package_file)

lib.nix

@@ -135,6 +135,22 @@ let
     })
   );
 
+  fetchFromLegacy = lib.makeOverridable (
+    { python, pname, url, file, hash }:
+    pkgs.runCommand file
+      {
+        nativeBuildInputs = [ python ];
+        impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+        SSL_CERT_FILE = "${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt";
+        outputHashMode = "flat";
+        outputHashAlgo = "sha256";
+        outputHash = hash;
+      } ''
+      python ${./fetch_from_legacy.py} ${url} ${pname} ${file}
+      mv ${file} $out
+    ''
+  );
+
   getBuildSystemPkgs =
     { pythonPackages
     , pyProject
@@ -201,6 +217,7 @@ in
 {
   inherit
     fetchFromPypi
+    fetchFromLegacy
     getManyLinuxDeps
     isCompatible
     readTOML

mk-poetry-dep.nix

@@ -28,7 +28,7 @@ pythonPackages.callPackage
     }@args:
     let
       inherit (pkgs) stdenv;
-      inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi moduleName;
+      inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi moduleName;
 
       inherit (import ./pep425.nix {
         inherit lib poetryLib python;
@@ -48,6 +48,7 @@ pythonPackages.callPackage
       isGit = isSource && source.type == "git";
       isUrl = isSource && source.type == "url";
       isLocal = isSource && source.type == "directory";
+      isLegacy = isSource && source.type == "legacy";
       localDepPath = toPath source.url;
 
       buildSystemPkgs =
@@ -171,6 +172,14 @@ pythonPackages.callPackage
             }
         else if isLocal then
           (poetryLib.cleanPythonSources { src = localDepPath; })
+        else if isLegacy then
+          fetchFromLegacy
+            {
+              pname = name;
+              inherit python;
+              inherit (fileInfo) file hash;
+              inherit (source) url;
+            }
         else
           fetchFromPypi {
             pname = name;

tests/default.nix

@@ -20,6 +20,7 @@ in
 builtins.removeAttrs
 {
   trivial = callTest ./trivial { };
+  legacy = callTest ./legacy { };
   composable-defaults = callTest ./composable-defaults { };
   override = callTest ./override-support { };
   override-default = callTest ./override-default-support { };

tests/legacy/default.nix

@@ -0,0 +1,6 @@
+{ lib, poetry2nix, python3 }:
+
+poetry2nix.mkPoetryApplication {
+  python = python3;
+  projectDir = ./.;
+}

tests/legacy/poetry.lock

@@ -0,0 +1,28 @@
+[[package]]
+name = "urllib3"
+version = "1.26.2"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
+
+[package.extras]
+brotli = ["brotlipy (>=0.6.0)"]
+secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+
+[package.source]
+type = "legacy"
+url = "https://pypi.org/simple"
+reference = "legacy"
+
+[metadata]
+lock-version = "1.1"
+python-versions = "^3.8"
+content-hash = "5014d201acf2a4a48d576c8a57bc4db2a5b3b2dac338a362a807a828420a8a12"
+
+[metadata.files]
+urllib3 = [
+    {file = "urllib3-1.26.2-py2.py3-none-any.whl", hash = "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"},
+    {file = "urllib3-1.26.2.tar.gz", hash = "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08"},
+]

tests/legacy/pyproject.toml

@@ -0,0 +1,20 @@
+[tool.poetry]
+name = "legacy"
+version = "0.1.0"
+description = "poetry2nix test"
+authors = ["Your Name <you@example.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+urllib3 = "1.26.2"
+
+# This is to force to use the Pypi legacy API
+[[tool.poetry.source]]
+name = "legacy"
+url = "https://pypi.org/simple/"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"