From 3f72aebfae8073be04a772f50d1b75521ebc20a9 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 12 Jul 2023 13:23:03 +1200 Subject: [PATCH] Redefine Github Actions in Nix using nix-github-actions --- .github/ci.nix | 43 ++++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 41 ++++++++++-------------------------- nix/sources.json | 12 +++++++++++ tests/default.nix | 45 +++++++++++++++++----------------------- 4 files changed, 85 insertions(+), 56 deletions(-) create mode 100644 .github/ci.nix diff --git a/.github/ci.nix b/.github/ci.nix new file mode 100644 index 0000000..d79daea --- /dev/null +++ b/.github/ci.nix @@ -0,0 +1,43 @@ +let + sources = import ../nix/sources.nix; + + inherit (import sources.nix-github-actions) mkGithubMatrix; + + mkPkgs = system: import sources.nixpkgs { + config = { + allowAliases = false; + allowInsecurePredicate = x: true; + }; + overlays = [ + (import ../overlay.nix) + ]; + inherit system; + }; + +in +mkGithubMatrix { + attrPrefix = ""; + checks = { + x86_64-linux = + let + pkgs = mkPkgs "x86_64-linux"; + in + import ../tests { inherit pkgs; }; + + x86_64-darwin = + let + pkgs = mkPkgs "x86_64-darwin"; + inherit (pkgs) lib; + + + tests = import ../tests { inherit pkgs; }; + in + { + # Aggregate all tests into one derivation so that only one GHA runner is scheduled for all darwin jobs + aggregate = pkgs.runCommand "darwin-aggregate" + { + env.TEST_INPUTS = (lib.concatStringsSep " " (lib.attrValues (lib.filterAttrs (n: v: lib.isDerivation v) tests))); + } "touch $out"; + }; + }; +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8abb50..4531d33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,58 +43,39 @@ jobs: - name: Check format run: nix-shell --arg packages 'pkgs:[ pkgs.p2nix-tools.env ]' --run 'black --check .' - matrix_generate: + nix-matrix: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@v3 - uses: cachix/install-nix-action@v22 - with: - nix_path: nixpkgs=channel:nixos-unstable - id: set-matrix + name: Generate Nix Matrix run: | - set -euo pipefail + set -Eeu + echo "matrix=$(nix eval --json -f ./.github/ci.nix matrix)" >> "$GITHUB_OUTPUT" - matrix="$(nix-instantiate --eval --json --expr 'builtins.attrNames (import ./tests {})' | jq -rcM '{attr: .}')" - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - - builds-linux: - needs: matrix_generate - runs-on: ubuntu-latest + nix-build: + needs: nix-matrix + runs-on: ${{ matrix.os }} strategy: - matrix: ${{fromJSON(needs.matrix_generate.outputs.matrix)}} + matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} steps: - uses: actions/checkout@v3 - uses: cachix/install-nix-action@v22 - with: - nix_path: nixpkgs=channel:nixos-unstable - uses: cachix/cachix-action@v12 with: name: poetry2nix signingKey: "VhaWuN3IyJVpWg+aZvTocVB+W8ziZKKRGLKR53Pkld3YRZxYOUfXZf0fvqF+LkqVW0eA60trVd5vsqNONpX9Hw==" - - run: nix-build --keep-going --show-trace tests/default.nix -A '${{ matrix.attr }}' - - builds-macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v22 - with: - nix_path: nixpkgs=channel:nixos-unstable - - uses: cachix/cachix-action@v12 - with: - name: poetry2nix - signingKey: "VhaWuN3IyJVpWg+aZvTocVB+W8ziZKKRGLKR53Pkld3YRZxYOUfXZf0fvqF+LkqVW0eA60trVd5vsqNONpX9Hw==" - - run: nix-build --keep-going --show-trace tests/default.nix -A '${{ matrix.attr }}' + - run: nix-build --keep-going --show-trace .github/ci.nix -A '${{ matrix.attr }}' collect: runs-on: ubuntu-latest needs: - nixpkgs-fmt - black-fmt - - builds-linux - - builds-macos + - nix-build - sort-build-systems steps: - - run: exit 0 + - run: true diff --git a/nix/sources.json b/nix/sources.json index 4df9dfd..acb21a0 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,4 +1,16 @@ { + "nix-github-actions": { + "branch": "master", + "description": "A library to turn Nix Flake attribute sets into Github Actions matrices [maintainer=@adisbladis]", + "homepage": "", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301", + "sha256": "0rkh2dcnl4rna4snqd7cdnvddjyqrgj3rais0dkhs4ygz64j8ip1", + "type": "tarball", + "url": "https://github.com/nix-community/nix-github-actions/archive/165b1650b753316aa7f1787f3005a8d2da0f5301.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "nixpkgs": { "branch": "master", "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", diff --git a/tests/default.nix b/tests/default.nix index 9097a48..de1ba59 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -18,14 +18,11 @@ let pep425 = pkgs.callPackage ../pep425.nix { inherit poetryLib; python = pkgs.python3; }; pep425PythonOldest = pkgs.callPackage ../pep425.nix { inherit poetryLib; python = pkgs.python38; }; pep425OSX = pkgs.callPackage ../pep425.nix { inherit poetryLib; isLinux = false; python = pkgs.python3; }; - skipTests = builtins.filter (t: builtins.typeOf t != "list") (builtins.split "," (builtins.getEnv "SKIP_TESTS")); callTest = test: attrs: pkgs.callPackage test ({ inherit poetry2nix; } // attrs); - # HACK: Return null on MacOS since the test in question fails - skipOSX = drv: if pkgs.stdenv.isDarwin then builtins.trace "Note: Skipping ${drv.name} on OSX" (pkgs.runCommand drv.name { } "touch $out") else drv; + inherit (pkgs) lib stdenv; in -builtins.removeAttrs { trivial = callTest ./trivial { }; @@ -39,7 +36,6 @@ builtins.removeAttrs common-pkgs-1 = callTest ./common-pkgs-1 { }; common-pkgs-2 = callTest ./common-pkgs-2 { }; pep425 = pkgs.callPackage ./pep425 { inherit pep425; inherit pep425OSX; inherit pep425PythonOldest; }; - pep600 = skipOSX (callTest ./pep600 { }); env = callTest ./env { }; pytest-metadata = callTest ./pytest-metadata { }; pytest-randomly = callTest ./pytest-randomly { }; @@ -62,25 +58,16 @@ builtins.removeAttrs inherit poetry; inherit (pkgs) postgresql; }; - # pyqt5 = skipOSX (callTest ./pyqt5 { }); extras = callTest ./extras { }; source-filter = callTest ./source-filter { }; canonical-module-names = callTest ./canonical-module-names { }; wandb = callTest ./wandb { }; utf8-pyproject = callTest ./utf8-pyproject { }; - # Test deadlocks on darwin, sandboxing issue? - dependency-environment = skipOSX (callTest ./dependency-environment { }); - - # Editable tests fails on Darwin because of sandbox paths - editable = skipOSX (callTest ./editable { }); - editable-egg = skipOSX (callTest ./editable-egg { }); - ansible-molecule = callTest ./ansible-molecule { }; bcrypt = callTest ./bcrypt { }; mk-poetry-packages = callTest ./mk-poetry-packages { }; markupsafe2 = callTest ./markupsafe2 { }; - pendulum = skipOSX (callTest ./pendulum { }); # uwsgi = callTest ./uwsgi { }; # Commented out because build is flaky (unrelated to poetry2nix) jq = callTest ./jq { }; ubersmith = callTest ./ubersmith { }; @@ -94,12 +81,6 @@ builtins.removeAttrs watchfiles = callTest ./watchfiles { }; sqlalchemy = callTest ./sqlalchemy { }; tzlocal = callTest ./tzlocal { }; - text-generation-webui = skipOSX (callTest ./text-generation-webui { }); - - # Cross tests fail on darwin for some strange reason: - # ERROR: MarkupSafe-2.0.1-cp39-cp39-linux_aarch64.whl is not a supported wheel on this platform. - extended-cross = skipOSX (callTest ./extended-cross { }); - trivial-cross = skipOSX (callTest ./trivial-cross { }); ml-stack = callTest ./ml-stack { }; @@ -171,11 +152,23 @@ builtins.removeAttrs cairocffi-no-wheel = callTest ./cairocffi-no-wheel { }; rpds-py = callTest ./rpds-py { }; - # Currently broken - # pandas = callTest ./pandas { }; - # Inherit test cases from nixpkgs - # nixops = pkgs.nixops; - nixops_unstable = skipOSX pkgs.nixops_unstable; +} // lib.optionalAttrs (!stdenv.isDarwin) { + # pyqt5 = (callTest ./pyqt5 { }); + # Test deadlocks on darwin, sandboxing issue? + dependency-environment = (callTest ./dependency-environment { }); + + # Editable tests fails on Darwin because of sandbox paths + pep600 = (callTest ./pep600 { }); + editable = (callTest ./editable { }); + editable-egg = (callTest ./editable-egg { }); + pendulum = (callTest ./pendulum { }); + + # Fails because of missing inputs on darwin + text-generation-webui = callTest ./text-generation-webui { }; + + # Cross tests fail on darwin for some strange reason: + # ERROR: MarkupSafe-2.0.1-cp39-cp39-linux_aarch64.whl is not a supported wheel on this platform. + extended-cross = (callTest ./extended-cross { }); + trivial-cross = (callTest ./trivial-cross { }); } - skipTests