Add CLI to supplement git hashes

This is not required for private repos where builtins.fetchGit would
suffice but _is_ required for Hydra (nixpkgs) where builtins.fetchGit is not allowed.

bin/poetry2nix

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+from concurrent.futures import ThreadPoolExecutor
+import subprocess
+import textwrap
+import argparse
+import toml
+import json
+import sys
+
+
+argparser = argparse.ArgumentParser(description="Generate overrides for git hashes",)
+argparser.add_argument(
+    "--lock", default="poetry.lock", help="Path to input poetry.lock",
+)
+argparser.add_argument(
+    "--out", default="poetry-git-overlay.nix", help="Output file",
+)
+
+
+def fetch_git(pkg):
+    return (
+        pkg["name"],
+        subprocess.run(
+            [
+                "nix-prefetch-git",
+                "--fetch-submodules",
+                "--url",
+                pkg["source"]["url"],
+                "--rev",
+                pkg["source"]["reference"],
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        ),
+    )
+
+
+def indent(expr, spaces=2):
+    i = " " * spaces
+    return "\n".join([(i if l != "" else "") + l for l in expr.split("\n")])
+
+
+if __name__ == "__main__":
+    args = argparser.parse_args()
+
+    with open(args.lock) as lockf:
+        lock = toml.load(lockf)
+
+    pkgs = []
+    for pkg in lock["package"]:
+        if "source" in pkg:
+            pkgs.append(pkg)
+
+    with ThreadPoolExecutor() as e:
+        futures = []
+
+        for pkg in pkgs:
+            futures.append(e.submit(fetch_git, pkg))
+
+        lines = [
+            "{ pkgs }:",
+            "self: super: {",
+        ]
+
+        for f in futures:
+            drv_name, p = f.result()
+            if p.returncode != 0:
+                sys.stderr.buffer.write(p.stderr)
+                sys.stderr.buffer.flush()
+                exit(p.returncode)
+
+            meta = json.loads(p.stdout.decode())
+            lines.append(
+                indent(
+                    textwrap.dedent(
+                        """
+              %s = super.%s.overrideAttrs (
+                _: {
+                  src = pkgs.fetchgit {
+                    url = "%s";
+                    rev = "%s";
+                    sha256 = "%s";
+                  };
+                }
+              );"""
+                        % (drv_name, drv_name, meta["url"], meta["rev"], meta["sha256"])
+                    )
+                )
+            )
+
+        lines.extend(["", "}", ""])
+
+        expr = "\n".join(lines)
+
+    with open(args.out, "w") as f:
+        f.write(expr)
+
+    print(f"Wrote {args.out}")

cli.nix

@@ -0,0 +1,51 @@
+{ pkgs ? import <nixpkgs> {}
+, lib ? pkgs.lib
+}:
+
+let
+  inherit (pkgs) python3;
+
+in
+pkgs.stdenv.mkDerivation {
+  pname = "poetry2nix";
+  version = "0.1";
+
+  buildInputs = [
+    (python3.withPackages (ps: [ ps.toml ]))
+  ];
+
+  nativeBuildInputs = [
+    pkgs.makeWrapper
+  ];
+
+  src = ./bin;
+
+  dontConfigure = true;
+
+  buildPhase = ''
+    runHook preBuild
+    ${python3.pkgs.black}/bin/black --quiet --check poetry2nix
+    patchShebangs poetry2nix
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+    mkdir -p $out/bin
+    mv poetry2nix $out/bin
+
+    wrapProgram $out/bin/poetry2nix --prefix PATH ":" ${lib.makeBinPath [
+    pkgs.nix-prefetch-git
+  ]}
+
+    runHook postInstall
+  '';
+
+  meta = {
+    homepage = "https://github.com/nix-community/poetry2nix";
+    description = "CLI to supplement sha256 hashes for git dependencies";
+    license = lib.licenses.mit;
+    maintainers = [ lib.maintainers.adisbladis ];
+  };
+
+}

default.nix

@@ -190,8 +190,11 @@ let
 
         }
       );
+
+  cli = import ./cli.nix { inherit pkgs lib; };
+
 in
 {
-  inherit mkPoetryPython mkPoetryEnv mkPoetryApplication defaultPoetryOverrides;
+  inherit mkPoetryPython mkPoetryEnv mkPoetryApplication defaultPoetryOverrides cli;
   mkPoetryPackage = attrs: builtins.trace "mkPoetryPackage is deprecated. Use mkPoetryApplication instead." (mkPoetryApplication attrs);
 }

tests/default.nix

@@ -16,6 +16,9 @@ in
   pep425 = pkgs.callPackage ./pep425 { inherit pep425; inherit pep425OSX; inherit pep425Python37; };
   env = pkgs.callPackage ./env { inherit poetry2nix; };
   git-deps = pkgs.callPackage ./git-deps { inherit poetry2nix; };
+  git-deps-pinned = pkgs.callPackage ./git-deps-pinned { inherit poetry2nix; };
+
+  cli = poetry2nix;
 
   # manylinux requires nixpkgs with https://github.com/NixOS/nixpkgs/pull/75763
   # Once this is available in 19.09 and unstable we can re-enable the manylinux test

tests/git-deps-pinned/default.nix

@@ -0,0 +1,13 @@
+{ pkgs, lib, poetry2nix, python3 }:
+
+poetry2nix.mkPoetryApplication {
+  python = python3;
+  pyproject = ./pyproject.toml;
+  poetrylock = ./poetry.lock;
+  src = lib.cleanSource ./.;
+
+  overrides = poetry2nix.defaultPoetryOverrides ++ [
+    (import ./poetry-git-overlay.nix { inherit pkgs; })
+  ];
+
+}

tests/git-deps-pinned/poetry-git-overlay.nix

@@ -0,0 +1,14 @@
+{ pkgs }:
+self: super: {
+
+  alembic = super.alembic.overrideAttrs (
+    _: {
+      src = pkgs.fetchgit {
+        url = "https://github.com/sqlalchemy/alembic.git";
+        rev = "8d6bb007a4de046c4d338f4b79b40c9fcbf73ab7";
+        sha256 = "15q4dsn4b1cjf1a4cxymxl2gzdjnv9zlndk98jmpfhssqsr4ky3w";
+      };
+    }
+  );
+
+}

tests/git-deps-pinned/poetry.lock

@@ -0,0 +1,141 @@
+[[package]]
+category = "main"
+description = "A database migration tool for SQLAlchemy."
+name = "alembic"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "1.3.1.dev0"
+
+[package.dependencies]
+Mako = "*"
+SQLAlchemy = ">=1.1.0"
+python-dateutil = "*"
+python-editor = ">=0.3"
+
+[package.source]
+reference = "8d6bb007a4de046c4d338f4b79b40c9fcbf73ab7"
+type = "git"
+url = "https://github.com/sqlalchemy/alembic.git"
+[[package]]
+category = "main"
+description = "A super-fast templating language that borrows the  best ideas from the existing templating languages."
+name = "mako"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "1.1.0"
+
+[package.dependencies]
+MarkupSafe = ">=0.9.2"
+
+[[package]]
+category = "main"
+description = "Safely add untrusted strings to HTML/XML markup."
+name = "markupsafe"
+optional = false
+python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"
+version = "1.1.1"
+
+[[package]]
+category = "main"
+description = "Extensions to the standard Python datetime module"
+name = "python-dateutil"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+version = "2.8.1"
+
+[package.dependencies]
+six = ">=1.5"
+
+[[package]]
+category = "main"
+description = "Programmatically open an editor, capture the result."
+name = "python-editor"
+optional = false
+python-versions = "*"
+version = "1.0.4"
+
+[[package]]
+category = "main"
+description = "Python 2 and 3 compatibility utilities"
+name = "six"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*"
+version = "1.13.0"
+
+[[package]]
+category = "main"
+description = "Database Abstraction Library"
+name = "sqlalchemy"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "1.3.11"
+
+[package.extras]
+mssql = ["pyodbc"]
+mssql_pymssql = ["pymssql"]
+mssql_pyodbc = ["pyodbc"]
+mysql = ["mysqlclient"]
+oracle = ["cx-oracle"]
+postgresql = ["psycopg2"]
+postgresql_pg8000 = ["pg8000"]
+postgresql_psycopg2binary = ["psycopg2-binary"]
+postgresql_psycopg2cffi = ["psycopg2cffi"]
+pymysql = ["pymysql"]
+
+[metadata]
+content-hash = "4ee5274ee8fd845e735f1bd2daa10da836f29bd7a2f90fe701c0b5f92b5a1438"
+python-versions = "^3.6"
+
+[metadata.files]
+alembic = []
+mako = [
+    {file = "Mako-1.1.0.tar.gz", hash = "sha256:a36919599a9b7dc5d86a7a8988f23a9a3a3d083070023bab23d64f7f1d1e0a4b"},
+]
+markupsafe = [
+    {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"},
+    {file = "MarkupSafe-1.1.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7"},
+    {file = "MarkupSafe-1.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183"},
+    {file = "MarkupSafe-1.1.1-cp27-cp27m-win32.whl", hash = "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b"},
+    {file = "MarkupSafe-1.1.1-cp27-cp27m-win_amd64.whl", hash = "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e"},
+    {file = "MarkupSafe-1.1.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f"},
+    {file = "MarkupSafe-1.1.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1"},
+    {file = "MarkupSafe-1.1.1-cp34-cp34m-macosx_10_6_intel.whl", hash = "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5"},
+    {file = "MarkupSafe-1.1.1-cp34-cp34m-manylinux1_i686.whl", hash = "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1"},
+    {file = "MarkupSafe-1.1.1-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735"},
+    {file = "MarkupSafe-1.1.1-cp34-cp34m-win32.whl", hash = "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21"},
+    {file = "MarkupSafe-1.1.1-cp34-cp34m-win_amd64.whl", hash = "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235"},
+    {file = "MarkupSafe-1.1.1-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b"},
+    {file = "MarkupSafe-1.1.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f"},
+    {file = "MarkupSafe-1.1.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905"},
+    {file = "MarkupSafe-1.1.1-cp35-cp35m-win32.whl", hash = "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1"},
+    {file = "MarkupSafe-1.1.1-cp35-cp35m-win_amd64.whl", hash = "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d"},
+    {file = "MarkupSafe-1.1.1-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff"},
+    {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473"},
+    {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e"},
+    {file = "MarkupSafe-1.1.1-cp36-cp36m-win32.whl", hash = "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66"},
+    {file = "MarkupSafe-1.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5"},
+    {file = "MarkupSafe-1.1.1-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d"},
+    {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e"},
+    {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"},
+    {file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"},
+    {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"},
+    {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
+]
+python-dateutil = [
+    {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"},
+    {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"},
+]
+python-editor = [
+    {file = "python-editor-1.0.4.tar.gz", hash = "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b"},
+    {file = "python_editor-1.0.4-py2-none-any.whl", hash = "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8"},
+    {file = "python_editor-1.0.4-py2.7.egg", hash = "sha256:ea87e17f6ec459e780e4221f295411462e0d0810858e055fc514684350a2f522"},
+    {file = "python_editor-1.0.4-py3-none-any.whl", hash = "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d"},
+    {file = "python_editor-1.0.4-py3.5.egg", hash = "sha256:c3da2053dbab6b29c94e43c486ff67206eafbe7eb52dbec7390b5e2fb05aac77"},
+]
+six = [
+    {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"},
+    {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"},
+]
+sqlalchemy = [
+    {file = "SQLAlchemy-1.3.11.tar.gz", hash = "sha256:afa5541e9dea8ad0014251bc9d56171ca3d8b130c9627c6cb3681cff30be3f8a"},
+]

tests/git-deps-pinned/pyproject.toml

@@ -0,0 +1,13 @@
+[tool.poetry]
+name = "git-deps"
+version = "0.1.0"
+description = "poetry2nix test"
+authors = ["Your Name <you@example.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.6"
+alembic = { git = "https://github.com/sqlalchemy/alembic.git", tag = "rel_1_3_1" }
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"