Merge pull request #433 from cpcloud/remove-pip-restriction

fix(pip): remove version restriction

default.nix

@@ -211,7 +211,7 @@ lib.makeScope pkgs.newScope (self: {
 
                   __toPluginAble = toPluginAble self;
 
-                  inherit (hooks) pipBuildHook removePathDependenciesHook poetry2nixFixupHook wheelUnpackHook;
+                  inherit (hooks) pipBuildHook removePathDependenciesHook removeGitDependenciesHook poetry2nixFixupHook wheelUnpackHook;
                 } // lib.optionalAttrs (! super ? setuptools-scm) {
                   # The canonical name is setuptools-scm
                   setuptools-scm = super.setuptools_scm;
@@ -313,7 +313,10 @@ lib.makeScope pkgs.newScope (self: {
 
       app = py.pkgs.buildPythonPackage (
         passedAttrs // inputAttrs // {
-          nativeBuildInputs = inputAttrs.nativeBuildInputs ++ [ py.pkgs.removePathDependenciesHook ];
+          nativeBuildInputs = inputAttrs.nativeBuildInputs ++ [
+            py.pkgs.removePathDependenciesHook
+            py.pkgs.removeGitDependenciesHook
+          ];
         } // {
           pname = moduleName pyProject.tool.poetry.name;
           version = pyProject.tool.poetry.version;

hooks/default.nix

@@ -21,9 +21,28 @@ in
           substitutions = {
             inherit pythonInterpreter;
             yj = "${buildPackages.yj}/bin/yj";
-            pyprojectPatchScript = "${./pyproject-without-path.py}";
+            pyprojectPatchScript = "${./pyproject-without-special-deps.py}";
+            fields = [ "path" ];
+            kind = "path";
           };
-        } ./remove-path-dependencies.sh
+        } ./remove-special-dependencies.sh
+    )
+    { };
+
+  removeGitDependenciesHook = callPackage
+    ({}:
+      makeSetupHook
+        {
+          name = "remove-git-dependencies.sh";
+          deps = [ ];
+          substitutions = {
+            inherit pythonInterpreter;
+            yj = "${buildPackages.yj}/bin/yj";
+            pyprojectPatchScript = "${./pyproject-without-special-deps.py}";
+            fields = [ "git" "branch" "rev" "tag" ];
+            kind = "git";
+          };
+        } ./remove-special-dependencies.sh
     )
     { };
 

hooks/pyproject-without-path.py

@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-# Patch out path dependencies from a pyproject.json file
-
-import json
-import sys
-
-data = json.load(sys.stdin)
-
-
-def get_deep(o, path):
-    for p in path.split('.'):
-        o = o.get(p, {})
-    return o
-
-
-for dep in get_deep(data, 'tool.poetry.dependencies').values():
-    if isinstance(dep, dict):
-        try:
-            del dep['path'];
-        except KeyError:
-            pass
-        else:
-            dep['version'] = '*'
-
-json.dump(data, sys.stdout, indent=4)

hooks/pyproject-without-special-deps.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# Patch out special dependencies (git and path) from a pyproject.json file
+
+import argparse
+import json
+import sys
+
+
+def main(input, output, fields_to_remove):
+    data = json.load(input)
+
+    try:
+        deps = data["tool"]["poetry"]["dependencies"]
+    except KeyError:
+        pass
+    else:
+        for dep in deps.values():
+            if isinstance(dep, dict):
+                any_removed = False
+                for field in fields_to_remove:
+                    any_removed |= dep.pop(field, None) is not None
+                if any_removed:
+                    dep["version"] = "*"
+
+    json.dump(data, output, separators=(",", ":"))
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser()
+    p.add_argument(
+        "-i",
+        "--input",
+        type=argparse.FileType("r"),
+        default=sys.stdin,
+        help="Location from which to read input JSON",
+    )
+    p.add_argument(
+        "-o",
+        "--output",
+        type=argparse.FileType("w"),
+        default=sys.stdout,
+        help="Location to write output JSON",
+    )
+    p.add_argument(
+        "-f",
+        "--fields-to-remove",
+        nargs="+",
+        help="The fields to remove from the dependency's JSON",
+    )
+
+    args = p.parse_args()
+    main(args.input, args.output, args.fields_to_remove)

hooks/remove-path-dependencies.sh

@@ -1,12 +0,0 @@
-remove-path-dependencies-hook() {
-    if ! test -f pyproject.toml; then
-        return
-    fi
-
-    # Tell poetry not to resolve the path dependencies. Any version is fine!
-    @yj@ -tj < pyproject.toml | @pythonInterpreter@ @pyprojectPatchScript@ > pyproject.json
-    @yj@ -jt < pyproject.json > pyproject.toml
-    rm pyproject.json
-}
-
-postPatchHooks+=(remove-path-dependencies-hook)

hooks/remove-special-dependencies.sh

@@ -0,0 +1,20 @@
+remove-@kind@-dependencies-hook() {
+    if ! test -f pyproject.toml; then
+        return
+    fi
+
+    echo "Removing @kind@ dependencies"
+
+    # Tell poetry not to resolve special dependencies. Any version is fine!
+    @yj@ -tj < pyproject.toml | \
+        @pythonInterpreter@ \
+        @pyprojectPatchScript@ \
+        --fields-to-remove @fields@ > pyproject.json
+    @yj@ -jt < pyproject.json > pyproject.toml
+
+    rm pyproject.json
+
+    echo "Finished removing @kind@ dependencies"
+}
+
+postPatchHooks+=(remove-@kind@-dependencies-hook)

mk-poetry-dep.nix

@@ -119,8 +119,10 @@ pythonPackages.callPackage
         pythonPackages.poetry2nixFixupHook
       ]
       ++ lib.optional (!isSource && (getManyLinuxDeps fileInfo.name).str != null) autoPatchelfHook
-      ++ lib.optional (format == "pyproject") pythonPackages.removePathDependenciesHook
+      ++ lib.optionals (format == "pyproject") [
-      ;
+        pythonPackages.removePathDependenciesHook
+        pythonPackages.removeGitDependenciesHook
+      ];
 
       buildInputs = (
         baseBuildInputs

overrides.nix

@@ -994,28 +994,6 @@ self: super:
     }
   );
 
-  # Work around https://github.com/nix-community/poetry2nix/issues/244
-  # where git deps are not picked up as they should
-  pip =
-    if lib.versionAtLeast super.pip.version "20.3" then
-      super.pip.overridePythonAttrs
-        (old:
-          let
-            pname = "pip";
-            version = "20.2.4";
-          in
-          {
-            name = pname + "-" + version;
-            inherit version;
-            src = pkgs.fetchFromGitHub {
-              owner = "pypa";
-              repo = pname;
-              rev = version;
-              sha256 = "eMVV4ftgV71HLQsSeaOchYlfaJVgzNrwUynn3SA1/Do=";
-              name = "${pname}-${version}-source";
-            };
-          }) else super.pip;
-
   platformdirs = super.platformdirs.overridePythonAttrs (old: {
     postPatch = ''
       substituteInPlace setup.py --replace 'setup()' 'setup(version="${old.version}")'

tests/common-pkgs-2/pyproject.toml

@@ -9,6 +9,7 @@ python = "^3.6"
 awscli-cwlogs = "^1.4.6"
 pytest = "^5.3.1"
 pyparsing = "^2.4.5"
+cryptography = "^3"
 decorator = "^4.4.1"
 flask = "^1.1.1"
 pluggy = "^0.13.1"
@@ -37,14 +38,24 @@ prompt-toolkit = "^3.0.0"
 atomicwrites = "^1.3.0"
 docker = "^4.1.0"
 importlib-metadata = "^0.23"
-argparse = "^1.4.0"
+
+# pip >=20.3 hardcodes its ignorance of a few modules that clash with stdlib
+# modules; one of those modules is argparse, so it can only be used where it's
+# actually required, otherwise pip thinks it's never installed
+#
+# Here's the hardcoded list:
+# https://github.com/pypa/pip/blob/e6891135776732d137ef3591d58d0de624521628/src/pip/_internal/utils/compat.py#L59
+#
+# Here's the code that skips over the list:
+# https://github.com/pypa/pip/blob/e6891135776732d137ef3591d58d0de624521628/src/pip/_internal/metadata/base.py#L194-L222
+argparse = { version = "^1.4.0", python = "<2.7" }
+
 packaging = "^19.2"
 zipp = "^0.6.0"
 google-resumable-media = "^0.5.0"
 multidict = "^4.6.1"
 sqlalchemy = "^1.3.11"
 kiwisolver = "^1.1.0"
-# mccabe = "^0.6.1"
 funcsigs = "^1.0.2"
 configparser = "^4.0.2"
 tornado = "^6.0.3"

tests/default.nix

@@ -24,8 +24,8 @@ builtins.removeAttrs
   composable-defaults = callTest ./composable-defaults { };
   override = callTest ./override-support { };
   override-default = callTest ./override-default-support { };
-  top-packages-1 = callTest ./common-pkgs-1 { };
+  common-pkgs-1 = callTest ./common-pkgs-1 { };
-  top-packages-2 = callTest ./common-pkgs-2 { };
+  common-pkgs-2 = callTest ./common-pkgs-2 { };
   pep425 = pkgs.callPackage ./pep425 { inherit pep425;inherit pep425OSX;inherit pep425Python37; };
   env = callTest ./env { };
   pytest-randomly = callTest ./pytest-randomly { };