Merge pull request #525 from nix-community/derived-overrides

overrides: Move build-systems to separate file which can be automatically generated
This commit is contained in:
adisbladis 2022-01-21 14:39:44 +12:00 committed by GitHub
commit cfccf191ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 3230 additions and 2269 deletions

View file

@ -22,7 +22,7 @@ jobs:
nix_path: nixpkgs=channel:nixos-unstable nix_path: nixpkgs=channel:nixos-unstable
- uses: actions/checkout@v2.4.0 - uses: actions/checkout@v2.4.0
- name: Check format - name: Check format
run: nix-shell --run 'git ls-files | grep -P "\.py$" | xargs py2-astparse' run: nix-shell --run 'git ls-files | grep -Pv "^tools" | grep -P "\.py$" | xargs py2-astparse'
black-fmt: black-fmt:
runs-on: ubuntu-latest runs-on: ubuntu-latest

View file

@ -192,7 +192,7 @@ poetry2nix.mkPoetryEditablePackage {
### defaultPoetryOverrides ### defaultPoetryOverrides
_poetry2nix_ bundles a set of default overrides that fix problems with various Python packages. These overrides are implemented in [overrides.nix](./overrides.nix). _poetry2nix_ bundles a set of default overrides that fix problems with various Python packages. These overrides are implemented in [overrides](./overrides/default.nix).
### overrides.withDefaults ### overrides.withDefaults
Returns a list containing the specified overlay and `defaultPoetryOverrides`. Returns a list containing the specified overlay and `defaultPoetryOverrides`.

View file

@ -437,7 +437,7 @@ lib.makeScope pkgs.newScope (self: {
Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function
*/ */
defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides.nix { inherit pkgs lib; }); defaultPoetryOverrides = self.mkDefaultPoetryOverrides (import ./overrides { inherit pkgs lib; });
/* /*
Convenience functions for specifying overlays with or without the poerty2nix default overrides Convenience functions for specifying overlays with or without the poerty2nix default overrides

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,830 @@
{
"adguardhome": [
"poetry-core"
],
"aioambient": [
"poetry-core"
],
"aiocurrencylayer": [
"poetry-core"
],
"aioeafm": [
"poetry"
],
"aiofiles": [
"poetry-core"
],
"aioflo": [
"poetry-core"
],
"aioguardian": [
"poetry-core"
],
"aiohomekit": [
"poetry-core"
],
"aiojobs": [
"flitBuildHook"
],
"aiomultiprocess": [
"flit-core"
],
"aiomusiccast": [
"poetry-core"
],
"aionotion": [
"poetry-core"
],
"aiopvpc": [
"poetry-core"
],
"aiorecollect": [
"poetry-core"
],
"aioresponses": [
"pbr"
],
"aioridwell": [
"poetry-core"
],
"aiorun": [
"flitBuildHook"
],
"aiosqlite": [
"poetry-core"
],
"aioswitcher": [
"poetry-core"
],
"aiowatttime": [
"poetry-core"
],
"ambee": [
"poetry-core"
],
"amqtt": [
"poetry-core"
],
"aria2p": [
"poetry"
],
"async-dns": [
"poetry-core"
],
"asyncmy": [
"poetry-core"
],
"asyncstdlib": [
"flitBuildHook"
],
"authcaptureproxy": [
"poetry-core"
],
"backcall": [
"flit-core"
],
"backoff": [
"poetry"
],
"bandit": [
"pbr"
],
"bash_kernel": [
"flitBuildHook"
],
"bimmer-connected": [
"pbr"
],
"build": [
"flit-core"
],
"cattrs": [
"poetry-core"
],
"censys": [
"poetry-core"
],
"cleo": [
"poetry-core"
],
"cliff": [
"pbr"
],
"clikit": [
"poetry-core"
],
"collections-extended": [
"poetry-core"
],
"colorclass": [
"poetry"
],
"colorhash": [
"poetry"
],
"confuse": [
"flit-core",
"flitBuildHook"
],
"crashtest": [
"poetry-core"
],
"credis": [
"poetry-core"
],
"cssselect2": [
"flit"
],
"cyclonedx-python-lib": [
"poetry-core"
],
"debtcollector": [
"pbr"
],
"deezer-python": [
"poetry-core"
],
"diagrams": [
"poetry-core"
],
"django-graphiql-debug-toolbar": [
"poetry-core"
],
"django-timezone-field": [
"poetry-core"
],
"doc8": [
"pbr"
],
"ecs-logging": [
"flit-core",
"flitBuildHook"
],
"eebrightbox": [
"pbr"
],
"elegy": [
"poetry"
],
"elgato": [
"poetry-core"
],
"elkm1-lib": [
"poetry-core"
],
"elmax": [
"poetry-core"
],
"enturclient": [
"poetry-core"
],
"fastapi": [
"flitBuildHook"
],
"fixtures": [
"pbr"
],
"flipr-api": [
"poetry-core"
],
"flit": [
"flit-core"
],
"formbox": [
"flitBuildHook"
],
"freebox-api": [
"poetry-core"
],
"funcparserlib": [
"poetry-core"
],
"furo": [
"flitBuildHook"
],
"gaphas": [
"poetry-core"
],
"generic": [
"poetry-core"
],
"glances-api": [
"poetry-core"
],
"gradient-utils": [
"poetry-core"
],
"graphql-core": [
"poetry-core"
],
"graphql-relay": [
"poetry-core"
],
"gvm-tools": [
"poetry-core"
],
"hacking": [
"pbr"
],
"hashids": [
"flit-core"
],
"hdate": [
"poetry-core"
],
"html5lib": [
"flit-core"
],
"hyperion-py": [
"poetry-core"
],
"hypothesis-auto": [
"poetry"
],
"hypothesis-graphql": [
"poetry"
],
"icmplib": [
"pbr"
],
"idasen": [
"poetry-core"
],
"immutabledict": [
"poetry-core"
],
"ipfshttpclient": [
"flitBuildHook"
],
"isort": [
"poetry-core"
],
"jedi-language-server": [
"poetry"
],
"jeepney": [
"flit-core"
],
"jenkins-job-builder": [
"pbr"
],
"jinja2-git": [
"poetry-core"
],
"jschema-to-python": [
"pbr"
],
"json-schema-for-humans": [
"poetry-core"
],
"keystoneauth1": [
"pbr"
],
"langcodes": [
"poetry-core"
],
"language-data": [
"poetry-core"
],
"lc7001": [
"poetry-core"
],
"ldappool": [
"pbr"
],
"linecache2": [
"pbr"
],
"loca": [
"flitBuildHook"
],
"lockfile": [
"pbr"
],
"luftdaten": [
"poetry-core"
],
"maestral": [
"pbr"
],
"matrix-nio": [
"poetry-core"
],
"mcstatus": [
"poetry-core"
],
"mdformat": [
"poetry-core"
],
"mdurl": [
"flit-core"
],
"mediafile": [
"flit-core",
"flitBuildHook"
],
"mock": [
"pbr"
],
"mongomock": [
"pbr"
],
"motioneye-client": [
"poetry-core"
],
"mox3": [
"pbr"
],
"msoffcrypto-tool": [
"poetry-core"
],
"munch": [
"pbr"
],
"myhome": [
"poetry-core"
],
"mypy-boto3-builder": [
"poetry-core"
],
"nats-python": [
"poetry-core"
],
"netdata": [
"poetry-core"
],
"nixpkgs": [
"pbr"
],
"notus-scanner": [
"poetry-core"
],
"ntc-templates": [
"poetry-core"
],
"omnikinverter": [
"poetry-core"
],
"open-meteo": [
"poetry-core"
],
"openstacksdk": [
"pbr"
],
"ormar": [
"poetry-core"
],
"os-service-types": [
"pbr"
],
"osc-lib": [
"pbr"
],
"oslo-concurrency": [
"pbr"
],
"oslo-config": [
"pbr"
],
"oslo-context": [
"pbr"
],
"oslo-db": [
"pbr"
],
"oslo-i18n": [
"pbr"
],
"oslo-log": [
"pbr"
],
"oslo-serialization": [
"pbr"
],
"oslo-utils": [
"pbr"
],
"oslotest": [
"pbr"
],
"p1monitor": [
"poetry-core"
],
"pastel": [
"poetry-core"
],
"pendulum": [
"poetry-core"
],
"pep440-version-utils": [
"poetry"
],
"pep517": [
"flit-core"
],
"pex": [
"flit-core",
"flitBuildHook"
],
"pixelmatch": [
"poetry-core"
],
"poetry": [
"poetry-core"
],
"poetry2conda": [
"poetry"
],
"pontos": [
"poetry-core"
],
"ptyprocess": [
"flit-core"
],
"pvo": [
"poetry-core"
],
"py-synologydsm-api": [
"poetry-core"
],
"py17track": [
"poetry-core"
],
"pyairnow": [
"poetry"
],
"pyairvisual": [
"poetry-core"
],
"pyebus": [
"poetry-core"
],
"pyflunearyou": [
"poetry-core"
],
"pyiqvia": [
"poetry-core"
],
"pyjsg": [
"pbr"
],
"pykka": [
"poetry-core"
],
"pymaven-patch": [
"pbr"
],
"pymfy": [
"poetry-core"
],
"pynixutil": [
"poetry"
],
"pynuki": [
"poetry-core"
],
"pyopenuv": [
"poetry-core"
],
"pyopnsense": [
"pbr"
],
"pypass": [
"pbr"
],
"pypika-tortoise": [
"poetry-core"
],
"pypoolstation": [
"poetry-core"
],
"pyppeteer": [
"poetry-core"
],
"pyproject-flake8": [
"flit-core"
],
"pyquil": [
"poetry-core"
],
"pyrmvtransport": [
"flit"
],
"pyshex": [
"pbr"
],
"pyshexc": [
"pbr"
],
"pysnow": [
"poetry"
],
"pytest-check": [
"flitBuildHook"
],
"pytest-cid": [
"flitBuildHook"
],
"pytest-httpserver": [
"poetry-core"
],
"pytest-mockservers": [
"poetry-core"
],
"pytest-raisin": [
"flit-core",
"flitBuildHook"
],
"pytest-socket": [
"poetry-core"
],
"python-awair": [
"poetry-core"
],
"python-cinderclient": [
"pbr"
],
"python-glanceclient": [
"pbr"
],
"python-gvm": [
"poetry-core"
],
"python-heatclient": [
"pbr"
],
"python-ironicclient": [
"pbr"
],
"python-jenkins": [
"pbr"
],
"python-kasa": [
"poetry-core"
],
"python-keystoneclient": [
"pbr"
],
"python-manilaclient": [
"pbr"
],
"python-miio": [
"poetry"
],
"python-novaclient": [
"pbr"
],
"python-openstackclient": [
"pbr"
],
"python-songpal": [
"poetry-core"
],
"python-swiftclient": [
"pbr"
],
"pytile": [
"poetry-core"
],
"pytzdata": [
"poetry"
],
"pyvera": [
"poetry-core"
],
"pywbem": [
"pbr"
],
"pywemo": [
"poetry-core"
],
"qcs-api-client": [
"poetry-core"
],
"questionary": [
"poetry-core"
],
"regenmaschine": [
"poetry-core"
],
"remarshal": [
"poetry-core"
],
"renault-api": [
"poetry-core"
],
"requests-cache": [
"poetry-core"
],
"requests-mock": [
"pbr"
],
"requests-unixsocket": [
"pbr"
],
"requestsexceptions": [
"pbr"
],
"retry": [
"pbr"
],
"rich": [
"poetry-core"
],
"rmcl": [
"poetry-core"
],
"rmfuse": [
"poetry-core"
],
"rmrl": [
"poetry-core"
],
"roombapy": [
"poetry-core"
],
"roonapi": [
"poetry-core"
],
"rsa": [
"poetry-core"
],
"sanic": [
"poetry-core"
],
"sarif-om": [
"pbr"
],
"shexjsg": [
"pbr"
],
"simplisafe-python": [
"poetry-core"
],
"slowapi": [
"poetry-core"
],
"solo-python": [
"flitBuildHook"
],
"sparqlslurper": [
"pbr"
],
"sphinx-inline-tabs": [
"flitBuildHook"
],
"sphinx-jinja": [
"pbr"
],
"sphinxcontrib-fulltoc": [
"pbr"
],
"sphinxcontrib-spelling": [
"pbr"
],
"spiderpy": [
"poetry-core"
],
"sqlalchemy-migrate": [
"pbr"
],
"sqlalchemy_migrate": [
"pbr"
],
"ssdp": [
"pbr"
],
"staticjinja": [
"poetry-core"
],
"stestr": [
"pbr"
],
"stevedore": [
"pbr"
],
"structlog": [
"flitBuildHook"
],
"subunit2sql": [
"pbr"
],
"surepy": [
"poetry-core"
],
"swift": [
"pbr"
],
"synologydsm-api": [
"poetry-core"
],
"tailscale": [
"poetry-core"
],
"tarsafe": [
"poetry"
],
"tempest": [
"pbr"
],
"tenacity": [
"pbr"
],
"terminaltables": [
"poetry"
],
"tern": [
"pbr"
],
"tesla-wall-connector": [
"poetry-core"
],
"teslajsonpy": [
"poetry-core"
],
"testrepository": [
"pbr"
],
"testresources": [
"pbr"
],
"testtools": [
"pbr"
],
"threadpoolctl": [
"flitBuildHook"
],
"tinycss2": [
"flitBuildHook"
],
"tinydb": [
"poetry-core"
],
"toggl-cli": [
"pbr"
],
"tomli": [
"flit-core"
],
"tomli-w": [
"flit-core"
],
"tomlkit": [
"poetry-core"
],
"tortoise-orm": [
"poetry-core"
],
"traceback2": [
"pbr"
],
"treeo": [
"poetry-core"
],
"treex": [
"poetry-core"
],
"twentemilieu": [
"poetry-core"
],
"typical": [
"poetry-core"
],
"typing-extensions": [
"flit-core"
],
"unpaddedbase64": [
"poetry-core"
],
"url-normalize": [
"poetry-core"
],
"vehicle": [
"poetry-core"
],
"virtualenvwrapper": [
"pbr"
],
"wakeonlan": [
"poetry-core"
],
"weasyprint": [
"flit-core"
],
"wled": [
"poetry-core"
],
"xdg": [
"poetry-core"
],
"xpath-expressions": [
"poetry-core"
],
"yaspin": [
"poetry-core"
],
"zeversolarlocal": [
"flit-core",
"flitBuildHook"
]
}

2218
overrides/default.nix Normal file

File diff suppressed because it is too large Load diff

View file

@ -22,5 +22,6 @@ pkgs.mkShell {
pkgs.niv pkgs.niv
pkgs.jq pkgs.jq
pkgs.nix-prefetch-git pkgs.nix-prefetch-git
pkgs.nix-eval-jobs
]; ];
} }

164
tools/find-build-systems.py Normal file
View file

@ -0,0 +1,164 @@
#!/usr/bin/env python
from concurrent.futures import ThreadPoolExecutor
from itertools import chain
from posix import cpu_count
from typing import (
Dict,
List,
Set,
)
import subprocess
import pynixutil
import tempfile
import json
import sys
# While we (im)patiently await a fix for https://github.com/python-poetry/poetry/pull/2794
# and require overrides to fix the issue we can at least regain a little bit of automation by inheriting these from nixpkgs
#
# This script evaluates nixpkgs and extracts a a few well known build systems and dumps them in a json file we can consume in the poetry2nix overrides
# All known PEP-517 build systems
BUILD_SYSTEMS = [
"poetry",
"poetry-core",
"flit",
"flit-core",
"pbr",
"flitBuildHook",
]
# Skip these attributes as they have more complex conditions manually
SKIP_ATTRS = {
"typing-extensions",
"argon2-cffi",
"packaging",
"poetry",
"flitBuildHook",
}
def find_known_systems() -> Dict[str, str]:
"""Create a map from attribute to drvPath for known build systems"""
expr = """let
pkgs = import <nixpkgs> { };
py = pkgs.python3.pkgs;
attrs = [ %s ];
in builtins.foldl' (
acc: attr: acc // {
${attr} = py.${attr}.drvPath;
}
) { } attrs""" % " ".join(
f'"{s}"' for s in BUILD_SYSTEMS
)
p = subprocess.run(
["nix-instantiate", "--eval", "--expr", f"builtins.toJSON ({expr})"],
stdout=subprocess.PIPE,
)
return json.loads(json.loads(p.stdout))
def yield_drvs():
"""Yield all drvs from the python3 set"""
with tempfile.NamedTemporaryFile(mode="w") as f:
f.write(
"""
let
pkgs = import <nixpkgs> { };
pythonPackages = pkgs.python3.pkgs;
in builtins.removeAttrs pythonPackages [
"pkgs"
"pythonPackages"
"__splicedPackages"
]
"""
)
f.flush()
p = subprocess.Popen(
["nix-eval-jobs", "--workers", str(cpu_count()), f.name],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
)
for l in p.stdout:
j = json.loads(l)
if "error" in j:
continue
try:
yield (j["attr"], j["drvPath"])
except KeyError:
pass
returncode = p.wait()
if returncode != 0:
raise ValueError(f"Eval returned: {returncode}")
def get_build_systems(known_systems) -> Dict[str, List[str]]:
def check_drv(drv_path) -> List[str]:
systems: List[str] = []
with open(drv_path) as f:
drv = pynixutil.drvparse(f.read())
input_drvs: Set[str] = set(drv.input_drvs.keys())
for attr, build_system in known_systems.items():
if build_system in input_drvs:
systems.append(attr)
return systems
with ThreadPoolExecutor() as e:
futures = {
attr: e.submit(check_drv, drv_path)
for attr, drv_path in yield_drvs()
if attr not in SKIP_ATTRS
}
build_systems = {attr: future.result() for attr, future in futures.items()}
# Second pass, filter out any empty lists
return {attr: systems for attr, systems in build_systems.items() if systems}
def merge(prev_content, new_content):
content = {}
for attr, systems in chain(prev_content.items(), new_content.items()):
s = content.setdefault(attr, set())
for system in systems:
s.add(system)
# Return with sorted data
return {attr: sorted(content[attr]) for attr in sorted(content.keys())}
def main():
outfile = sys.argv[1]
try:
with open(outfile) as f:
prev_content = json.load(f)
except FileNotFoundError:
prev_content = {}
known_systems = find_known_systems()
build_systems = get_build_systems(known_systems)
# Unlike nixpkgs we want overrides to be strictly additive by
# merging content from previous generations with new generations
merged = merge(prev_content, build_systems)
with open(outfile, mode="w") as f:
json.dump(merged, f, indent=2)
f.write("\n")
if __name__ == "__main__":
main()

14
tools/poetry.lock generated
View file

@ -71,6 +71,14 @@ python-versions = ">=3.7"
docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"]
test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"]
[[package]]
name = "pynixutil"
version = "0.5.0"
description = "Utility functions for working with Nix in Python"
category = "main"
optional = false
python-versions = ">=3.8,<4.0"
[[package]] [[package]]
name = "tomli" name = "tomli"
version = "1.2.3" version = "1.2.3"
@ -90,7 +98,7 @@ python-versions = ">=3.6"
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = "^3.9" python-versions = "^3.9"
content-hash = "a5ad398e01214edd1b90cc904036e0521c150e983480a329c352b02f178386c5" content-hash = "5611bf0fe9c11eca8d20586ec493a789fbe2a1eb6314395ad9d4733234dd7d50"
[metadata.files] [metadata.files]
black = [ black = [
@ -117,6 +125,10 @@ platformdirs = [
{file = "platformdirs-2.4.1-py3-none-any.whl", hash = "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca"}, {file = "platformdirs-2.4.1-py3-none-any.whl", hash = "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca"},
{file = "platformdirs-2.4.1.tar.gz", hash = "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda"}, {file = "platformdirs-2.4.1.tar.gz", hash = "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda"},
] ]
pynixutil = [
{file = "pynixutil-0.5.0-py3-none-any.whl", hash = "sha256:81b933e0a2f9b9950cbeb236893992fffd1f1696632e5130cfeffe37a7344215"},
{file = "pynixutil-0.5.0.tar.gz", hash = "sha256:62e86960d67fef68e77f6a24d90015b3573a71d163c163e2ebc313d96235d1ab"},
]
tomli = [ tomli = [
{file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"},
{file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"},

View file

@ -8,6 +8,7 @@ license = "MIT"
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.9" python = "^3.9"
black = "^21.12b0" black = "^21.12b0"
pynixutil = "^0.5.0"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]