Merge pull request #525 from nix-community/derived-overrides

overrides: Move build-systems to separate file which can be automatically generated
This commit is contained in:
adisbladis 2022-01-21 14:39:44 +12:00 committed by GitHub
commit cfccf191ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 3230 additions and 2269 deletions

View file

@ -22,7 +22,7 @@ jobs:
nix_path: nixpkgs=channel:nixos-unstable
- uses: actions/checkout@v2.4.0
- name: Check format
run: nix-shell --run 'git ls-files | grep -P "\.py$" | xargs py2-astparse'
run: nix-shell --run 'git ls-files | grep -Pv "^tools" | grep -P "\.py$" | xargs py2-astparse'
black-fmt:
runs-on: ubuntu-latest

View file

@ -192,7 +192,7 @@ poetry2nix.mkPoetryEditablePackage {
### defaultPoetryOverrides
_poetry2nix_ bundles a set of default overrides that fix problems with various Python packages. These overrides are implemented in [overrides.nix](./overrides.nix).
_poetry2nix_ bundles a set of default overrides that fix problems with various Python packages. These overrides are implemented in [overrides](./overrides/default.nix).
### overrides.withDefaults
Returns a list containing the specified overlay and `defaultPoetryOverrides`.

View file

@ -437,7 +437,7 @@ lib.makeScope pkgs.newScope (self: {
Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function
*/
defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides.nix { inherit pkgs lib; });
defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib; });
/*
Convenience functions for specifying overlays with or without the poerty2nix default overrides

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,830 @@
{
"adguardhome": [
"poetry-core"
],
"aioambient": [
"poetry-core"
],
"aiocurrencylayer": [
"poetry-core"
],
"aioeafm": [
"poetry"
],
"aiofiles": [
"poetry-core"
],
"aioflo": [
"poetry-core"
],
"aioguardian": [
"poetry-core"
],
"aiohomekit": [
"poetry-core"
],
"aiojobs": [
"flitBuildHook"
],
"aiomultiprocess": [
"flit-core"
],
"aiomusiccast": [
"poetry-core"
],
"aionotion": [
"poetry-core"
],
"aiopvpc": [
"poetry-core"
],
"aiorecollect": [
"poetry-core"
],
"aioresponses": [
"pbr"
],
"aioridwell": [
"poetry-core"
],
"aiorun": [
"flitBuildHook"
],
"aiosqlite": [
"poetry-core"
],
"aioswitcher": [
"poetry-core"
],
"aiowatttime": [
"poetry-core"
],
"ambee": [
"poetry-core"
],
"amqtt": [
"poetry-core"
],
"aria2p": [
"poetry"
],
"async-dns": [
"poetry-core"
],
"asyncmy": [
"poetry-core"
],
"asyncstdlib": [
"flitBuildHook"
],
"authcaptureproxy": [
"poetry-core"
],
"backcall": [
"flit-core"
],
"backoff": [
"poetry"
],
"bandit": [
"pbr"
],
"bash_kernel": [
"flitBuildHook"
],
"bimmer-connected": [
"pbr"
],
"build": [
"flit-core"
],
"cattrs": [
"poetry-core"
],
"censys": [
"poetry-core"
],
"cleo": [
"poetry-core"
],
"cliff": [
"pbr"
],
"clikit": [
"poetry-core"
],
"collections-extended": [
"poetry-core"
],
"colorclass": [
"poetry"
],
"colorhash": [
"poetry"
],
"confuse": [
"flit-core",
"flitBuildHook"
],
"crashtest": [
"poetry-core"
],
"credis": [
"poetry-core"
],
"cssselect2": [
"flit"
],
"cyclonedx-python-lib": [
"poetry-core"
],
"debtcollector": [
"pbr"
],
"deezer-python": [
"poetry-core"
],
"diagrams": [
"poetry-core"
],
"django-graphiql-debug-toolbar": [
"poetry-core"
],
"django-timezone-field": [
"poetry-core"
],
"doc8": [
"pbr"
],
"ecs-logging": [
"flit-core",
"flitBuildHook"
],
"eebrightbox": [
"pbr"
],
"elegy": [
"poetry"
],
"elgato": [
"poetry-core"
],
"elkm1-lib": [
"poetry-core"
],
"elmax": [
"poetry-core"
],
"enturclient": [
"poetry-core"
],
"fastapi": [
"flitBuildHook"
],
"fixtures": [
"pbr"
],
"flipr-api": [
"poetry-core"
],
"flit": [
"flit-core"
],
"formbox": [
"flitBuildHook"
],
"freebox-api": [
"poetry-core"
],
"funcparserlib": [
"poetry-core"
],
"furo": [
"flitBuildHook"
],
"gaphas": [
"poetry-core"
],
"generic": [
"poetry-core"
],
"glances-api": [
"poetry-core"
],
"gradient-utils": [
"poetry-core"
],
"graphql-core": [
"poetry-core"
],
"graphql-relay": [
"poetry-core"
],
"gvm-tools": [
"poetry-core"
],
"hacking": [
"pbr"
],
"hashids": [
"flit-core"
],
"hdate": [
"poetry-core"
],
"html5lib": [
"flit-core"
],
"hyperion-py": [
"poetry-core"
],
"hypothesis-auto": [
"poetry"
],
"hypothesis-graphql": [
"poetry"
],
"icmplib": [
"pbr"
],
"idasen": [
"poetry-core"
],
"immutabledict": [
"poetry-core"
],
"ipfshttpclient": [
"flitBuildHook"
],
"isort": [
"poetry-core"
],
"jedi-language-server": [
"poetry"
],
"jeepney": [
"flit-core"
],
"jenkins-job-builder": [
"pbr"
],
"jinja2-git": [
"poetry-core"
],
"jschema-to-python": [
"pbr"
],
"json-schema-for-humans": [
"poetry-core"
],
"keystoneauth1": [
"pbr"
],
"langcodes": [
"poetry-core"
],
"language-data": [
"poetry-core"
],
"lc7001": [
"poetry-core"
],
"ldappool": [
"pbr"
],
"linecache2": [
"pbr"
],
"loca": [
"flitBuildHook"
],
"lockfile": [
"pbr"
],
"luftdaten": [
"poetry-core"
],
"maestral": [
"pbr"
],
"matrix-nio": [
"poetry-core"
],
"mcstatus": [
"poetry-core"
],
"mdformat": [
"poetry-core"
],
"mdurl": [
"flit-core"
],
"mediafile": [
"flit-core",
"flitBuildHook"
],
"mock": [
"pbr"
],
"mongomock": [
"pbr"
],
"motioneye-client": [
"poetry-core"
],
"mox3": [
"pbr"
],
"msoffcrypto-tool": [
"poetry-core"
],
"munch": [
"pbr"
],
"myhome": [
"poetry-core"
],
"mypy-boto3-builder": [
"poetry-core"
],
"nats-python": [
"poetry-core"
],
"netdata": [
"poetry-core"
],
"nixpkgs": [
"pbr"
],
"notus-scanner": [
"poetry-core"
],
"ntc-templates": [
"poetry-core"
],
"omnikinverter": [
"poetry-core"
],
"open-meteo": [
"poetry-core"
],
"openstacksdk": [
"pbr"
],
"ormar": [
"poetry-core"
],
"os-service-types": [
"pbr"
],
"osc-lib": [
"pbr"
],
"oslo-concurrency": [
"pbr"
],
"oslo-config": [
"pbr"
],
"oslo-context": [
"pbr"
],
"oslo-db": [
"pbr"
],
"oslo-i18n": [
"pbr"
],
"oslo-log": [
"pbr"
],
"oslo-serialization": [
"pbr"
],
"oslo-utils": [
"pbr"
],
"oslotest": [
"pbr"
],
"p1monitor": [
"poetry-core"
],
"pastel": [
"poetry-core"
],
"pendulum": [
"poetry-core"
],
"pep440-version-utils": [
"poetry"
],
"pep517": [
"flit-core"
],
"pex": [
"flit-core",
"flitBuildHook"
],
"pixelmatch": [
"poetry-core"
],
"poetry": [
"poetry-core"
],
"poetry2conda": [
"poetry"
],
"pontos": [
"poetry-core"
],
"ptyprocess": [
"flit-core"
],
"pvo": [
"poetry-core"
],
"py-synologydsm-api": [
"poetry-core"
],
"py17track": [
"poetry-core"
],
"pyairnow": [
"poetry"
],
"pyairvisual": [
"poetry-core"
],
"pyebus": [
"poetry-core"
],
"pyflunearyou": [
"poetry-core"
],
"pyiqvia": [
"poetry-core"
],
"pyjsg": [
"pbr"
],
"pykka": [
"poetry-core"
],
"pymaven-patch": [
"pbr"
],
"pymfy": [
"poetry-core"
],
"pynixutil": [
"poetry"
],
"pynuki": [
"poetry-core"
],
"pyopenuv": [
"poetry-core"
],
"pyopnsense": [
"pbr"
],
"pypass": [
"pbr"
],
"pypika-tortoise": [
"poetry-core"
],
"pypoolstation": [
"poetry-core"
],
"pyppeteer": [
"poetry-core"
],
"pyproject-flake8": [
"flit-core"
],
"pyquil": [
"poetry-core"
],
"pyrmvtransport": [
"flit"
],
"pyshex": [
"pbr"
],
"pyshexc": [
"pbr"
],
"pysnow": [
"poetry"
],
"pytest-check": [
"flitBuildHook"
],
"pytest-cid": [
"flitBuildHook"
],
"pytest-httpserver": [
"poetry-core"
],
"pytest-mockservers": [
"poetry-core"
],
"pytest-raisin": [
"flit-core",
"flitBuildHook"
],
"pytest-socket": [
"poetry-core"
],
"python-awair": [
"poetry-core"
],
"python-cinderclient": [
"pbr"
],
"python-glanceclient": [
"pbr"
],
"python-gvm": [
"poetry-core"
],
"python-heatclient": [
"pbr"
],
"python-ironicclient": [
"pbr"
],
"python-jenkins": [
"pbr"
],
"python-kasa": [
"poetry-core"
],
"python-keystoneclient": [
"pbr"
],
"python-manilaclient": [
"pbr"
],
"python-miio": [
"poetry"
],
"python-novaclient": [
"pbr"
],
"python-openstackclient": [
"pbr"
],
"python-songpal": [
"poetry-core"
],
"python-swiftclient": [
"pbr"
],
"pytile": [
"poetry-core"
],
"pytzdata": [
"poetry"
],
"pyvera": [
"poetry-core"
],
"pywbem": [
"pbr"
],
"pywemo": [
"poetry-core"
],
"qcs-api-client": [
"poetry-core"
],
"questionary": [
"poetry-core"
],
"regenmaschine": [
"poetry-core"
],
"remarshal": [
"poetry-core"
],
"renault-api": [
"poetry-core"
],
"requests-cache": [
"poetry-core"
],
"requests-mock": [
"pbr"
],
"requests-unixsocket": [
"pbr"
],
"requestsexceptions": [
"pbr"
],
"retry": [
"pbr"
],
"rich": [
"poetry-core"
],
"rmcl": [
"poetry-core"
],
"rmfuse": [
"poetry-core"
],
"rmrl": [
"poetry-core"
],
"roombapy": [
"poetry-core"
],
"roonapi": [
"poetry-core"
],
"rsa": [
"poetry-core"
],
"sanic": [
"poetry-core"
],
"sarif-om": [
"pbr"
],
"shexjsg": [
"pbr"
],
"simplisafe-python": [
"poetry-core"
],
"slowapi": [
"poetry-core"
],
"solo-python": [
"flitBuildHook"
],
"sparqlslurper": [
"pbr"
],
"sphinx-inline-tabs": [
"flitBuildHook"
],
"sphinx-jinja": [
"pbr"
],
"sphinxcontrib-fulltoc": [
"pbr"
],
"sphinxcontrib-spelling": [
"pbr"
],
"spiderpy": [
"poetry-core"
],
"sqlalchemy-migrate": [
"pbr"
],
"sqlalchemy_migrate": [
"pbr"
],
"ssdp": [
"pbr"
],
"staticjinja": [
"poetry-core"
],
"stestr": [
"pbr"
],
"stevedore": [
"pbr"
],
"structlog": [
"flitBuildHook"
],
"subunit2sql": [
"pbr"
],
"surepy": [
"poetry-core"
],
"swift": [
"pbr"
],
"synologydsm-api": [
"poetry-core"
],
"tailscale": [
"poetry-core"
],
"tarsafe": [
"poetry"
],
"tempest": [
"pbr"
],
"tenacity": [
"pbr"
],
"terminaltables": [
"poetry"
],
"tern": [
"pbr"
],
"tesla-wall-connector": [
"poetry-core"
],
"teslajsonpy": [
"poetry-core"
],
"testrepository": [
"pbr"
],
"testresources": [
"pbr"
],
"testtools": [
"pbr"
],
"threadpoolctl": [
"flitBuildHook"
],
"tinycss2": [
"flitBuildHook"
],
"tinydb": [
"poetry-core"
],
"toggl-cli": [
"pbr"
],
"tomli": [
"flit-core"
],
"tomli-w": [
"flit-core"
],
"tomlkit": [
"poetry-core"
],
"tortoise-orm": [
"poetry-core"
],
"traceback2": [
"pbr"
],
"treeo": [
"poetry-core"
],
"treex": [
"poetry-core"
],
"twentemilieu": [
"poetry-core"
],
"typical": [
"poetry-core"
],
"typing-extensions": [
"flit-core"
],
"unpaddedbase64": [
"poetry-core"
],
"url-normalize": [
"poetry-core"
],
"vehicle": [
"poetry-core"
],
"virtualenvwrapper": [
"pbr"
],
"wakeonlan": [
"poetry-core"
],
"weasyprint": [
"flit-core"
],
"wled": [
"poetry-core"
],
"xdg": [
"poetry-core"
],
"xpath-expressions": [
"poetry-core"
],
"yaspin": [
"poetry-core"
],
"zeversolarlocal": [
"flit-core",
"flitBuildHook"
]
}

2218
overrides/default.nix Normal file

File diff suppressed because it is too large Load diff

View file

@ -22,5 +22,6 @@ pkgs.mkShell {
pkgs.niv
pkgs.jq
pkgs.nix-prefetch-git
pkgs.nix-eval-jobs
];
}

164
tools/find-build-systems.py Normal file
View file

@ -0,0 +1,164 @@
#!/usr/bin/env python
from concurrent.futures import ThreadPoolExecutor
from itertools import chain
from posix import cpu_count
from typing import (
Dict,
List,
Set,
)
import subprocess
import pynixutil
import tempfile
import json
import sys
# While we (im)patiently await a fix for https://github.com/python-poetry/poetry/pull/2794
# and require overrides to fix the issue we can at least regain a little bit of automation by inheriting these from nixpkgs
#
# This script evaluates nixpkgs and extracts a a few well known build systems and dumps them in a json file we can consume in the poetry2nix overrides
# All known PEP-517 build systems
BUILD_SYSTEMS = [
"poetry",
"poetry-core",
"flit",
"flit-core",
"pbr",
"flitBuildHook",
]
# Skip these attributes as they have more complex conditions manually
SKIP_ATTRS = {
"typing-extensions",
"argon2-cffi",
"packaging",
"poetry",
"flitBuildHook",
}
def find_known_systems() -> Dict[str, str]:
"""Create a map from attribute to drvPath for known build systems"""
expr = """let
pkgs = import <nixpkgs> { };
py = pkgs.python3.pkgs;
attrs = [ %s ];
in builtins.foldl' (
acc: attr: acc // {
${attr} = py.${attr}.drvPath;
}
) { } attrs""" % " ".join(
f'"{s}"' for s in BUILD_SYSTEMS
)
p = subprocess.run(
["nix-instantiate", "--eval", "--expr", f"builtins.toJSON ({expr})"],
stdout=subprocess.PIPE,
)
return json.loads(json.loads(p.stdout))
def yield_drvs():
"""Yield all drvs from the python3 set"""
with tempfile.NamedTemporaryFile(mode="w") as f:
f.write(
"""
let
pkgs = import <nixpkgs> { };
pythonPackages = pkgs.python3.pkgs;
in builtins.removeAttrs pythonPackages [
"pkgs"
"pythonPackages"
"__splicedPackages"
]
"""
)
f.flush()
p = subprocess.Popen(
["nix-eval-jobs", "--workers", str(cpu_count()), f.name],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
)
for l in p.stdout:
j = json.loads(l)
if "error" in j:
continue
try:
yield (j["attr"], j["drvPath"])
except KeyError:
pass
returncode = p.wait()
if returncode != 0:
raise ValueError(f"Eval returned: {returncode}")
def get_build_systems(known_systems) -> Dict[str, List[str]]:
def check_drv(drv_path) -> List[str]:
systems: List[str] = []
with open(drv_path) as f:
drv = pynixutil.drvparse(f.read())
input_drvs: Set[str] = set(drv.input_drvs.keys())
for attr, build_system in known_systems.items():
if build_system in input_drvs:
systems.append(attr)
return systems
with ThreadPoolExecutor() as e:
futures = {
attr: e.submit(check_drv, drv_path)
for attr, drv_path in yield_drvs()
if attr not in SKIP_ATTRS
}
build_systems = {attr: future.result() for attr, future in futures.items()}
# Second pass, filter out any empty lists
return {attr: systems for attr, systems in build_systems.items() if systems}
def merge(prev_content, new_content):
content = {}
for attr, systems in chain(prev_content.items(), new_content.items()):
s = content.setdefault(attr, set())
for system in systems:
s.add(system)
# Return with sorted data
return {attr: sorted(content[attr]) for attr in sorted(content.keys())}
def main():
outfile = sys.argv[1]
try:
with open(outfile) as f:
prev_content = json.load(f)
except FileNotFoundError:
prev_content = {}
known_systems = find_known_systems()
build_systems = get_build_systems(known_systems)
# Unlike nixpkgs we want overrides to be strictly additive by
# merging content from previous generations with new generations
merged = merge(prev_content, build_systems)
with open(outfile, mode="w") as f:
json.dump(merged, f, indent=2)
f.write("\n")
if __name__ == "__main__":
main()

14
tools/poetry.lock generated
View file

@ -71,6 +71,14 @@ python-versions = ">=3.7"
docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"]
test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"]
[[package]]
name = "pynixutil"
version = "0.5.0"
description = "Utility functions for working with Nix in Python"
category = "main"
optional = false
python-versions = ">=3.8,<4.0"
[[package]]
name = "tomli"
version = "1.2.3"
@ -90,7 +98,7 @@ python-versions = ">=3.6"
[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "a5ad398e01214edd1b90cc904036e0521c150e983480a329c352b02f178386c5"
content-hash = "5611bf0fe9c11eca8d20586ec493a789fbe2a1eb6314395ad9d4733234dd7d50"
[metadata.files]
black = [
@ -117,6 +125,10 @@ platformdirs = [
{file = "platformdirs-2.4.1-py3-none-any.whl", hash = "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca"},
{file = "platformdirs-2.4.1.tar.gz", hash = "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda"},
]
pynixutil = [
{file = "pynixutil-0.5.0-py3-none-any.whl", hash = "sha256:81b933e0a2f9b9950cbeb236893992fffd1f1696632e5130cfeffe37a7344215"},
{file = "pynixutil-0.5.0.tar.gz", hash = "sha256:62e86960d67fef68e77f6a24d90015b3573a71d163c163e2ebc313d96235d1ab"},
]
tomli = [
{file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"},
{file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"},

View file

@ -8,6 +8,7 @@ license = "MIT"
[tool.poetry.dependencies]
python = "^3.9"
black = "^21.12b0"
pynixutil = "^0.5.0"
[tool.poetry.dev-dependencies]