From 6014c45ac17faae12943936fb08613687d855f2e Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 8 Aug 2023 19:08:25 +0200 Subject: [PATCH 01/92] Add nix files --- .envrc | 11 ++ .gitignore | 2 + flake.lock | 376 +++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 45 +++++++ 4 files changed, 434 insertions(+) create mode 100644 .envrc create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..b351dff3d --- /dev/null +++ b/.envrc @@ -0,0 +1,11 @@ +if ! has nix_direnv_version || ! nix_direnv_version 2.2.1; then + source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.2.1/direnvrc" "sha256-zelF0vLbEl5uaqrfIzbgNzJWGmLzCmYAkInj/LNxvKs=" +fi + +nix_direnv_watch_file devenv.nix +nix_direnv_watch_file devenv.lock +nix_direnv_watch_file devenv.yaml +if ! use flake . --impure +then + echo "devenv could not be built. The devenv environment was not loaded. Make the necessary changes to devenv.nix and hit enter to try again." >&2 +fi diff --git a/.gitignore b/.gitignore index ea8c4bf7f..c305a71e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /target + +.devenv/ diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..5509f9b70 --- /dev/null +++ b/flake.lock @@ -0,0 +1,376 @@ +{ + "nodes": { + "devenv": { + "inputs": { + "flake-compat": "flake-compat", + "nix": "nix", + "nixpkgs": "nixpkgs", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1691413946, + "narHash": "sha256-IQ9gWBnUMJaXXev9vjnUJDfX0y4tOprGF773xD0rwHA=", + "owner": "cachix", + "repo": "devenv", + "rev": "b4ce82b6a54022c688f9f916b9f7a9c9ed959de1", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1691475627, + "narHash": "sha256-aOgYmNqXyCHw1vaNDv5GqUAURiPMVej4xQNE37bCF8g=", + "owner": "nix-community", + "repo": "fenix", + "rev": "72b64bd857cacd2733ce99fbf0603bbe335d357c", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1685518550, + "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1685518550, + "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "type": "github" + }, + "original": { + "id": "flake-utils", + "type": "indirect" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "devenv", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1660459072, + "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "lowdown-src": { + "flake": false, + "locked": { + "lastModified": 1633514407, + "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", + "owner": "kristapsdz", + "repo": "lowdown", + "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", + "type": "github" + }, + "original": { + "owner": "kristapsdz", + "repo": "lowdown", + "type": "github" + } + }, + "nix": { + "inputs": { + "lowdown-src": "lowdown-src", + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1676545802, + "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", + "owner": "domenkozar", + "repo": "nix", + "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "relaxed-flakes", + "repo": "nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1678875422, + "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-python": { + "inputs": { + "flake-compat": "flake-compat_2", + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1690216544, + "narHash": "sha256-fWZVYvyoGTNzRnr6MDd0O2FLA2GwLCkSOlZbOxEBuk0=", + "owner": "cachix", + "repo": "nixpkgs-python", + "rev": "64b32e82cba00c8a1fdc53618ae8b5502866fe96", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "nixpkgs-python", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1685801374, + "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1688392541, + "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1685974512, + "narHash": "sha256-WLPHpe96RbPRO9iDtCxgsYkadTheRq7wqXWdGpR6g7w=", + "owner": "domenkozar", + "repo": "nixpkgs", + "rev": "1102477695918daba466123cc2ef694ed3a49939", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "cpython-moduralize", + "repo": "nixpkgs", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": [ + "devenv", + "flake-compat" + ], + "flake-utils": "flake-utils", + "gitignore": "gitignore", + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1688056373, + "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "fenix": "fenix", + "nixpkgs": "nixpkgs_2", + "nixpkgs-python": "nixpkgs-python", + "systems": "systems_3" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1691410619, + "narHash": "sha256-npJ8zOZ/CQoZu+0kN5NU9iC5JbmbsucyK1DayPmHWPs=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "d9e9ca29814e95bb40db2216ed0973d9ba54b6a2", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..a2fafac27 --- /dev/null +++ b/flake.nix @@ -0,0 +1,45 @@ +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11"; + systems.url = "github:nix-systems/default"; + devenv.url = "github:cachix/devenv"; + nixpkgs-python.url = "github:cachix/nixpkgs-python"; + fenix.url = "github:nix-community/fenix"; + fenix.inputs.nixpkgs.follows = "nixpkgs"; + }; + + nixConfig = { + extra-trusted-public-keys = "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw="; + extra-substituters = "https://devenv.cachix.org"; + }; + + outputs = { self, nixpkgs, devenv, systems, ... } @ inputs: + let + forEachSystem = nixpkgs.lib.genAttrs (import systems); + in + { + devShells = forEachSystem + (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + default = devenv.lib.mkShell { + inherit inputs pkgs; + modules = [ + { + languages.python = { + enable = true; + venv.enable = true; + version = "3.11"; + }; + languages.rust = { + enable = true; + channel = "stable"; + }; + } + ]; + }; + }); + }; +} From 4df6ce6d70bab10b229b25aa3f6c67c5c2e0955b Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 11 Oct 2023 19:11:55 +0200 Subject: [PATCH 02/92] Add maturin, update nixos distribution --- flake.lock | 8 +++---- flake.nix | 67 +++++++++++++++++++++++++++++------------------------- 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/flake.lock b/flake.lock index 5509f9b70..3732b67e5 100644 --- a/flake.lock +++ b/flake.lock @@ -241,16 +241,16 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1688392541, - "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "lastModified": 1696697597, + "narHash": "sha256-q26Qv4DQ+h6IeozF2o1secyQG0jt2VUT3V0K58jr3pg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "rev": "5a237aecb57296f67276ac9ab296a41c23981f56", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.11", + "ref": "nixos-23.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a2fafac27..dc2b473c6 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,6 @@ { inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; systems.url = "github:nix-systems/default"; devenv.url = "github:cachix/devenv"; nixpkgs-python.url = "github:cachix/nixpkgs-python"; @@ -8,38 +8,43 @@ fenix.inputs.nixpkgs.follows = "nixpkgs"; }; + outputs = { + self, + nixpkgs, + devenv, + systems, + ... + } @ inputs: let + forEachSystem = nixpkgs.lib.genAttrs (import systems); + in { + devShells = + forEachSystem + (system: let + pkgs = nixpkgs.legacyPackages.${system}; + in { + default = devenv.lib.mkShell { + inherit inputs pkgs; + modules = [ + { + packages = with pkgs; [maturin]; + + languages.python = { + enable = true; + venv.enable = true; + version = "3.11"; + }; + languages.rust = { + enable = true; + channel = "stable"; + }; + } + ]; + }; + }); + }; + nixConfig = { extra-trusted-public-keys = "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw="; extra-substituters = "https://devenv.cachix.org"; }; - - outputs = { self, nixpkgs, devenv, systems, ... } @ inputs: - let - forEachSystem = nixpkgs.lib.genAttrs (import systems); - in - { - devShells = forEachSystem - (system: - let - pkgs = nixpkgs.legacyPackages.${system}; - in - { - default = devenv.lib.mkShell { - inherit inputs pkgs; - modules = [ - { - languages.python = { - enable = true; - venv.enable = true; - version = "3.11"; - }; - languages.rust = { - enable = true; - channel = "stable"; - }; - } - ]; - }; - }); - }; } From 936cd0cc312658efbd46ff16b709442baa004b0f Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Sat, 24 Feb 2024 15:20:54 +0100 Subject: [PATCH 03/92] Add lhapdf dependency, setup shell to ease its usage --- flake.nix | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index dc2b473c6..7dd8a1865 100644 --- a/flake.nix +++ b/flake.nix @@ -21,12 +21,25 @@ forEachSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; + pwd = builtins.getEnv "PWD"; + path = builtins.getEnv "PATH"; + prefix = "${pwd}/target/prefix"; + lhapath = "${prefix}/share/LHAPDF"; in { default = devenv.lib.mkShell { inherit inputs pkgs; modules = [ { - packages = with pkgs; [maturin]; + packages = with pkgs; [maturin lhapdf]; + + env = { + PREFIX = prefix; + PATH = "${path}:${prefix}/bin"; + LHAPDF_DATA_PATH = lhapath; + }; + enterShell = '' + mkdir -p ${lhapath} + ''; languages.python = { enable = true; From b657b74a4b44032f6ee696b5b7bc47dd6ad9670d Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Sun, 25 Feb 2024 10:25:30 +0100 Subject: [PATCH 04/92] Propagate Python version to the LHAPDF package --- flake.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 7dd8a1865..89d7c428c 100644 --- a/flake.nix +++ b/flake.nix @@ -29,8 +29,14 @@ default = devenv.lib.mkShell { inherit inputs pkgs; modules = [ - { - packages = with pkgs; [maturin lhapdf]; + ({config, ...}: { + packages = with pkgs; [ + maturin + (lhapdf.override { + python = + config.languages.python.package; + }) + ]; env = { PREFIX = prefix; @@ -50,7 +56,7 @@ enable = true; channel = "stable"; }; - } + }) ]; }; }); From 5d38e6425f144210ccdb0b240e50706a8e2f5cd2 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Sun, 25 Feb 2024 10:28:16 +0100 Subject: [PATCH 05/92] Use PATH variable from module system If taken directly from the environment, it will lose all the additions made by the Nix packages Accessing the value from the configuration will always improve self-consistency --- flake.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 89d7c428c..a979095fc 100644 --- a/flake.nix +++ b/flake.nix @@ -22,14 +22,15 @@ (system: let pkgs = nixpkgs.legacyPackages.${system}; pwd = builtins.getEnv "PWD"; - path = builtins.getEnv "PATH"; prefix = "${pwd}/target/prefix"; lhapath = "${prefix}/share/LHAPDF"; in { default = devenv.lib.mkShell { inherit inputs pkgs; modules = [ - ({config, ...}: { + ({config, ...}: let + path = config.system.path; + in { packages = with pkgs; [ maturin (lhapdf.override { From 87e52a8762adbeb8b1630c8544cb7d50f26ee712 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Sun, 25 Feb 2024 11:54:31 +0100 Subject: [PATCH 06/92] Drop path extension, temporarily --- flake.nix | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index a979095fc..bae3eb8a5 100644 --- a/flake.nix +++ b/flake.nix @@ -28,9 +28,7 @@ default = devenv.lib.mkShell { inherit inputs pkgs; modules = [ - ({config, ...}: let - path = config.system.path; - in { + ({config, ...}: { packages = with pkgs; [ maturin (lhapdf.override { @@ -41,7 +39,6 @@ env = { PREFIX = prefix; - PATH = "${path}:${prefix}/bin"; LHAPDF_DATA_PATH = lhapath; }; enterShell = '' From ca96f730bb1c245fc7d370dbacde7943c3d7a3b1 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Sun, 25 Feb 2024 12:55:20 +0100 Subject: [PATCH 07/92] Set pineappl PATH in the trivial way Just run a shell export command right before entering the shell --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index bae3eb8a5..ea008d414 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,9 @@ LHAPDF_DATA_PATH = lhapath; }; enterShell = '' + # update path before entering the shell, when Nix packages updates + # already happened + export PATH=${prefix}/bin:$PATH mkdir -p ${lhapath} ''; From ff404dbbbb372706e3a49104b61b172a3fa62132 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Sun, 25 Feb 2024 15:10:21 +0100 Subject: [PATCH 08/92] Document Nix files added to the repo --- .gitignore | 1 + docs/maintainers-guide.md | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index c305a71e7..cd52e6918 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target +# Devenv files, to handle the development shell defined by the flake .devenv/ diff --git a/docs/maintainers-guide.md b/docs/maintainers-guide.md index cd31d69c4..c9f493dfe 100644 --- a/docs/maintainers-guide.md +++ b/docs/maintainers-guide.md @@ -33,6 +33,7 @@ fastNLO tables to PineAPPL grids - `pineappl_py`: the crate that builds PineAPPL's Python interface - `xtask`: crate for [cargo-xtask] commands +- `.envrc`: [direnv] hooks (mainly enable Nix shell) - `.gitignore`: PineAPPL's Git ignore rules - `.readthedocs.yml`: configuration for PineAPPL's [Read-the-Docs] Python interface documentation @@ -44,6 +45,9 @@ - `README.md`: PineAPPL's repository README file. This file also serves as the homepage of [PineAPPL's website] - `_config.yml`: configuration file for PineAPPL's Github-pages website +- `flake.lock`: Nix [flake lock file] +- `flake.nix`: Nix [flake], which tracks cross-language dependencies to define a + reproducible build for the PineAPPL packages, and a suitable development shell - `install-capi.sh`: POSIX-compliant shell script to download and install PineAPPL's pre-built CAPI @@ -57,6 +61,9 @@ [lock file]: https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html [license v3]: https://www.gnu.org/licenses/gpl-3.0.en.html [PineAPPL's website]: https://nnpdf.github.io/pineappl/ +[flake]: https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html#flake-format +[flake lock file]: https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html#lock-files +[direnv]: https://direnv.net/ ## Abbreviations and terminology From 2a0567458f652a271250ed8305242407442318c4 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Thu, 21 Mar 2024 23:49:41 +0400 Subject: [PATCH 09/92] feat: Add Python CLI packaging workflow --- .github/workflows/cli-wheels.yml | 104 +++++++++++++++++++++++++++++++ pineappl_cli/pyproject.toml | 6 ++ 2 files changed, 110 insertions(+) create mode 100644 .github/workflows/cli-wheels.yml create mode 100644 pineappl_cli/pyproject.toml diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml new file mode 100644 index 000000000..70c83c7a1 --- /dev/null +++ b/.github/workflows/cli-wheels.yml @@ -0,0 +1,104 @@ +name: Release + +on: + push: + # tags: + # - "v[0-9]+*" + workflow_dispatch: + +env: + GH_TOKEN: ${{ github.token }} + +jobs: + wheels-linux: + runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64] + steps: + - uses: actions/checkout@v4 + # for Linux the wheels are built in a container, so we don't need the `setup-python` action + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + # `--find-interpreter` is needed to generate wheels for *all* Python versions + args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml + sccache: "true" + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-${{ matrix.target }} + path: dist + + wheels-macos: + runs-on: macos-latest + strategy: + matrix: + target: [aarch64, x86_64] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + # WARNING: be careful with the ordering - the last version is the default one. Apparently + # maturin doesn't find all Python versions when one chooses a PyPy version instead of + # CPython as default + python-version: | + pypy3.7 + pypy3.8 + pypy3.9 + pypy3.10 + 3.7 + 3.8 + 3.9 + 3.11 + 3.12 + 3.10 + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml + sccache: "true" + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-${{ matrix.target }} + path: dist + + wheels-windows: + runs-on: windows-latest + strategy: + matrix: + target: [x64] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + # WARNING: be careful with the ordering - the last version is the default one. Apparently + # maturin doesn't find all Python versions when one chooses a PyPy version instead of + # CPython as default + python-version: | + pypy3.7 + pypy3.8 + pypy3.9 + pypy3.10 + 3.7 + 3.8 + 3.9 + 3.11 + 3.12 + 3.10 + architecture: ${{ matrix.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml + sccache: "true" + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-${{ matrix.target }} + path: dist diff --git a/pineappl_cli/pyproject.toml b/pineappl_cli/pyproject.toml new file mode 100644 index 000000000..0988846e1 --- /dev/null +++ b/pineappl_cli/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[tool.maturin] +bindings = "bin" From 8ceef174da863a482c211b344bdd3c90fb3d2678 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 07:38:32 +0400 Subject: [PATCH 10/92] Switch from maturin action to manual maturin usage --- .github/workflows/cli-wheels.yml | 74 +++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 70c83c7a1..4a54cf8d5 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -12,20 +12,40 @@ env: jobs: wheels-linux: runs-on: ubuntu-latest + # TODO: maybe I just need manylinux + container: ghcr.io/nnpdf/pineappl-ci:latest strategy: matrix: target: [x86_64] + steps: - uses: actions/checkout@v4 - # for Linux the wheels are built in a container, so we don't need the `setup-python` action - - name: Build wheels - uses: PyO3/maturin-action@v1 + - uses: actions/setup-python@v5 with: - target: ${{ matrix.target }} - # `--find-interpreter` is needed to generate wheels for *all* Python versions - args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml - sccache: "true" - manylinux: auto + # WARNING: be careful with the ordering - the last version is the + # default one. Apparently maturin doesn't find all Python versions + # when one chooses a PyPy version instead of CPython as default + python-version: | + pypy3.7 + pypy3.8 + pypy3.9 + pypy3.10 + 3.7 + 3.8 + 3.9 + 3.11 + 3.12 + 3.10 + architecture: ${{ matrix.target }} + - name: Install build tools + run: | + pip install maturin auditwheel + - name: Build wheels + run: | + # `--find-interpreter` is needed to generate wheels for + # *all* Python versions + maturin build --release --out dist --find-interpreter \ + --manifest-path pineappl_cli/Cargo.toml - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -41,9 +61,9 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - # WARNING: be careful with the ordering - the last version is the default one. Apparently - # maturin doesn't find all Python versions when one chooses a PyPy version instead of - # CPython as default + # WARNING: be careful with the ordering - the last version is the + # default one. Apparently maturin doesn't find all Python versions + # when one chooses a PyPy version instead of CPython as default python-version: | pypy3.7 pypy3.8 @@ -55,12 +75,15 @@ jobs: 3.11 3.12 3.10 + - name: Install build tools + run: | + pip install maturin delocate - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml - sccache: "true" + run: | + # `--find-interpreter` is needed to generate wheels for + # *all* Python versions + maturin build --release --out dist --find-interpreter \ + --manifest-path pineappl_cli/Cargo.toml - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -76,9 +99,9 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - # WARNING: be careful with the ordering - the last version is the default one. Apparently - # maturin doesn't find all Python versions when one chooses a PyPy version instead of - # CPython as default + # WARNING: be careful with the ordering - the last version is the + # default one. Apparently maturin doesn't find all Python versions + # when one chooses a PyPy version instead of CPython as default python-version: | pypy3.7 pypy3.8 @@ -91,12 +114,15 @@ jobs: 3.12 3.10 architecture: ${{ matrix.target }} + - name: Install build tools + run: | + pip install maturin delvewheel - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml - sccache: "true" + run: | + # `--find-interpreter` is needed to generate wheels for + # *all* Python versions + maturin build --release --out dist --find-interpreter \ + --manifest-path pineappl_cli/Cargo.toml - name: Upload wheels uses: actions/upload-artifact@v4 with: From a52ad6a0bfb31494c5c0d02c453b4064f07e6b37 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 07:42:11 +0400 Subject: [PATCH 11/92] Patch CLI wheels with auditwheel and delocate --- .github/workflows/cli-wheels.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 4a54cf8d5..79a49c6f6 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -46,6 +46,9 @@ jobs: # *all* Python versions maturin build --release --out dist --find-interpreter \ --manifest-path pineappl_cli/Cargo.toml + - name: Patch wheels + run: | + auditwheel repair dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -84,6 +87,9 @@ jobs: # *all* Python versions maturin build --release --out dist --find-interpreter \ --manifest-path pineappl_cli/Cargo.toml + - name: Patch wheels + run: | + delocate-wheel dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: From 44948d659d81c41f1a5d63737e4d3f9dbad19951 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 09:11:55 +0400 Subject: [PATCH 12/92] Add LHAPDF installation in the CI --- .github/workflows/cli-wheels.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 79a49c6f6..8ab5782a1 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -40,6 +40,13 @@ jobs: - name: Install build tools run: | pip install maturin auditwheel + - name: Install non-Rust dependencies + run: | + for script in {install,download,variables}; do + curl \ + https://raw.githubusercontent.com/NNPDF/workflows/v2/packages/lhapdf/${script}.sh + done + sh install.sh - name: Build wheels run: | # `--find-interpreter` is needed to generate wheels for @@ -81,6 +88,13 @@ jobs: - name: Install build tools run: | pip install maturin delocate + - name: Install non-Rust dependencies + run: | + for script in {install,download,variables}; do + curl \ + https://raw.githubusercontent.com/NNPDF/workflows/v2/packages/lhapdf/${script}.sh + done + sh install.sh - name: Build wheels run: | # `--find-interpreter` is needed to generate wheels for @@ -123,6 +137,13 @@ jobs: - name: Install build tools run: | pip install maturin delvewheel + - name: Install non-Rust dependencies + run: | + for script in {install,download,variables}; do + curl \ + https://raw.githubusercontent.com/NNPDF/workflows/v2/packages/lhapdf/${script}.sh + done + sh install.sh - name: Build wheels run: | # `--find-interpreter` is needed to generate wheels for From 7e65f0d71284744e61a711d82ec96a5ca722d1b8 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 09:15:47 +0400 Subject: [PATCH 13/92] Set bash as default shell for all jobs --- .github/workflows/cli-wheels.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 8ab5782a1..c7cca8d93 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -6,14 +6,17 @@ on: # - "v[0-9]+*" workflow_dispatch: +defaults: + run: + shell: bash + env: GH_TOKEN: ${{ github.token }} jobs: wheels-linux: runs-on: ubuntu-latest - # TODO: maybe I just need manylinux - container: ghcr.io/nnpdf/pineappl-ci:latest + container: quay.io/pypa/manylinux2014_x86_64 strategy: matrix: target: [x86_64] @@ -67,6 +70,7 @@ jobs: strategy: matrix: target: [aarch64, x86_64] + steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -115,6 +119,7 @@ jobs: strategy: matrix: target: [x64] + steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From 3f423001961d823386a5e7a604a28560f4cf4ad8 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 09:21:31 +0400 Subject: [PATCH 14/92] Write curl output on file --- .github/workflows/cli-wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index c7cca8d93..634da1391 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -1,4 +1,4 @@ -name: Release +name: Publish CLI wheels on: push: @@ -46,7 +46,7 @@ jobs: - name: Install non-Rust dependencies run: | for script in {install,download,variables}; do - curl \ + curl -O \ https://raw.githubusercontent.com/NNPDF/workflows/v2/packages/lhapdf/${script}.sh done sh install.sh @@ -95,7 +95,7 @@ jobs: - name: Install non-Rust dependencies run: | for script in {install,download,variables}; do - curl \ + curl -O \ https://raw.githubusercontent.com/NNPDF/workflows/v2/packages/lhapdf/${script}.sh done sh install.sh @@ -145,7 +145,7 @@ jobs: - name: Install non-Rust dependencies run: | for script in {install,download,variables}; do - curl \ + curl -O \ https://raw.githubusercontent.com/NNPDF/workflows/v2/packages/lhapdf/${script}.sh done sh install.sh From 9f39dec4ec62127876edde54efc45bde51b6e947 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 10:34:25 +0400 Subject: [PATCH 15/92] Add LHAPDF installation script --- maintainer/install-lhapdf.py | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 maintainer/install-lhapdf.py diff --git a/maintainer/install-lhapdf.py b/maintainer/install-lhapdf.py new file mode 100644 index 000000000..66e13545a --- /dev/null +++ b/maintainer/install-lhapdf.py @@ -0,0 +1,63 @@ +"""Install LHAPDF.""" + +import io +import logging +import os +import sys +import tarfile +import tempfile +from http.client import HTTPSConnection +from pathlib import Path +from subprocess import run + +VERSION = "6.5.3" +TARBALL = f"LHAPDF-{VERSION}.tar.gz" +HOST = "lhapdf.hepforge.org" +PATH = f"/downloads/?f={TARBALL}" +SRC_DIR = f"LHAPDF-{VERSION}" + + +logging.basicConfig() +log = logging.getLogger("LHAPDF installer") +log.setLevel(logging.INFO) + + +def download(host: str, path: str) -> bytes: + """Download content from remote server.""" + conn = HTTPSConnection(host) + conn.request("GET", path) + resp = conn.getresponse() + return resp.read() + + +def install(path: Path): + """Build from source and install.""" + + def run_(cmd, *args, **kwargs): + cmd = cmd.split() if isinstance(cmd, str) else cmd + run(cmd, *args, cwd=path / SRC_DIR, **kwargs) + + run_("autoreconf -f -i") + prefix = os.environ.get("PREFIX") + prefix_ = ["--prefix", prefix] if prefix is not None else [] + run_(["./configure"] + prefix_, env={"PYTHON": sys.executable}) + log.info("Configured") + run_("make clean") + run_("make -j") + log.info("Built") + run_("make install") + log.info("Installed") + + +def main(): + """Install LHAPDF.""" + tar = download(HOST, PATH) + log.info(f"Downloaded {HOST}{PATH}") + with tempfile.TemporaryDirectory() as tmpd: + tarfile.open(fileobj=io.BytesIO(tar), mode="r:gz").extractall(tmpd) + log.info(f"Extracted LHAPDF tarbal in {tmpd}") + install(Path(tmpd)) + + +if __name__ == "__main__": + main() From 6175f4eee686a819898e1da90cceff8ad5185be3 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 10:35:37 +0400 Subject: [PATCH 16/92] Replace shell LHAPDF installation with Python-based one It ensures that the tools are available on every platform, since the script only makes use of Python standard library. Otherwise it was not even possible to rely on `wget` or `realpath`... --- .github/workflows/cli-wheels.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 634da1391..ef6b3b781 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -45,11 +45,7 @@ jobs: pip install maturin auditwheel - name: Install non-Rust dependencies run: | - for script in {install,download,variables}; do - curl -O \ - https://raw.githubusercontent.com/NNPDF/workflows/v2/packages/lhapdf/${script}.sh - done - sh install.sh + python maintainer/install-lhapdf.py - name: Build wheels run: | # `--find-interpreter` is needed to generate wheels for @@ -94,11 +90,7 @@ jobs: pip install maturin delocate - name: Install non-Rust dependencies run: | - for script in {install,download,variables}; do - curl -O \ - https://raw.githubusercontent.com/NNPDF/workflows/v2/packages/lhapdf/${script}.sh - done - sh install.sh + python maintainer/install-lhapdf.py - name: Build wheels run: | # `--find-interpreter` is needed to generate wheels for @@ -144,11 +136,7 @@ jobs: pip install maturin delvewheel - name: Install non-Rust dependencies run: | - for script in {install,download,variables}; do - curl -O \ - https://raw.githubusercontent.com/NNPDF/workflows/v2/packages/lhapdf/${script}.sh - done - sh install.sh + python maintainer/install-lhapdf.py - name: Build wheels run: | # `--find-interpreter` is needed to generate wheels for From 18bd0eb41a5c4155dcadbcd5c504c9a51295e397 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 10:40:42 +0400 Subject: [PATCH 17/92] Downgrade checkout action to run within container --- .github/workflows/cli-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index ef6b3b781..18cbe6613 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -22,7 +22,7 @@ jobs: target: [x86_64] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the From 3251aed43d88e940765caf52d5bd4cf4f8b41056 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 10:56:06 +0400 Subject: [PATCH 18/92] Downgrade setup python action as well --- .github/workflows/cli-wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 18cbe6613..ffd881c58 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -23,7 +23,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v4 with: # WARNING: be careful with the ordering - the last version is the # default one. Apparently maturin doesn't find all Python versions @@ -100,6 +100,7 @@ jobs: - name: Patch wheels run: | delocate-wheel dist/* + python maintainer/patch-macos-wheel.py - name: Upload wheels uses: actions/upload-artifact@v4 with: From ac45fbdf3043382c70f8b3b0649f140bb580c34e Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 11:49:25 +0400 Subject: [PATCH 19/92] Drop PyPy 3.7 on Linux Apparently not available, t least with the given container. --- .github/workflows/cli-wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index ffd881c58..9ba845606 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -29,7 +29,6 @@ jobs: # default one. Apparently maturin doesn't find all Python versions # when one chooses a PyPy version instead of CPython as default python-version: | - pypy3.7 pypy3.8 pypy3.9 pypy3.10 From f2a262fc9b08d00abbacccb56ca5f0b60b4264e1 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 12:02:20 +0400 Subject: [PATCH 20/92] Add MacOS wheel patching script --- maintainer/patch-macos.wheel.py | 87 +++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 maintainer/patch-macos.wheel.py diff --git a/maintainer/patch-macos.wheel.py b/maintainer/patch-macos.wheel.py new file mode 100644 index 000000000..f00be39ba --- /dev/null +++ b/maintainer/patch-macos.wheel.py @@ -0,0 +1,87 @@ +"""Patch MacOS wheel to allow executable binaries. + +Following the patch in https://github.com/pypa/auditwheel/pull/443/ +""" + +import base64 +import hashlib +import logging +import shutil +import sys +import tempfile +from pathlib import Path + +logging.basicConfig() +log = logging.getLogger("Patch MacOS wheels") +log.setLevel(logging.INFO) + +WRAPPER = """ +#!python +import os +import sys +import sysconfig +if __name__ == "__main__": + os.execv( + os.path.join(sysconfig.get_path("platlib"), {binary_path!r}), + sys.argv, + ) +""" + + +def hash(path: Path): + """Compute file hash.""" + return ( + base64.urlsafe_b64encode(hashlib.sha256(path.read_bytes()).digest()) + .decode() + .rstrip("=") + ) + + +def record(tmpd): + """Generate a RECORD file. + + Rehash the content of the wheel. + """ + lines = [] + for path in Path(tmpd).glob("**/*"): + if not path.is_file(): + continue + + path_ = path.relative_to(tmpd) + if path.name == "RECORD": + lines.append(f"{path_},,") + continue + + hash_ = hash(path) + size = path.stat().st_size + lines.append(f"{path_},sha256={hash_},{size}") + + return "\n".join(lines) + + +def patch(wheel: Path): + """Patch wheel to add wrapper script.""" + with tempfile.TemporaryDirectory() as tmpd: + shutil.unpack_archive(wheel, tmpd, format="zip") + scripts = Path(tmpd) / "pineappl_cli.scripts" + scripts.mkdir() + installed = next(Path(tmpd).glob("*.data")) / "scripts" / "pineappl" + shutil.move(installed, scripts) + binary = scripts / "pineappl" + installed.write_text(WRAPPER.format(binary_path=str(binary.relative_to(tmpd)))) + next(Path(tmpd).glob("**/RECORD")).write_text(record(tmpd)) + shutil.make_archive(wheel.name, format="zip", base_dir=".", root_dir=tmpd) + shutil.move(str(wheel) + ".zip", wheel) + log.info(f"Patched {wheel} in-place") + + +def main(): + """Patch wheels passed as arguments.""" + wheels = [Path(p) for p in sys.argv[1:]] + log.info("Processing the following wheels:\n" + "\n".join(str(w) for w in wheels)) + for wheel in wheels: + patch(wheel) + + +if __name__ == "__main__": + main() From b20d336af0b0f0c647e02a5bd20e0fc1fda80c57 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 12:11:10 +0400 Subject: [PATCH 21/92] Fix typo in script name --- maintainer/{patch-macos.wheel.py => patch-macos-wheel.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename maintainer/{patch-macos.wheel.py => patch-macos-wheel.py} (100%) diff --git a/maintainer/patch-macos.wheel.py b/maintainer/patch-macos-wheel.py similarity index 100% rename from maintainer/patch-macos.wheel.py rename to maintainer/patch-macos-wheel.py From e6c987fe34162d0998d6c9f878949e4487851309 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 12:31:58 +0400 Subject: [PATCH 22/92] Set suitable targets for macos --- .github/workflows/cli-wheels.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 9ba845606..b8081f0e7 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -92,10 +92,20 @@ jobs: python maintainer/install-lhapdf.py - name: Build wheels run: | + case ${{ matrix.target }} in + aarch64) + TARGET="aarch64-apple-darwin" + ;; + x86_64) + TARGET="x86_64-apple-darwin" + ;; + *) + exit 1 + esac # `--find-interpreter` is needed to generate wheels for # *all* Python versions maturin build --release --out dist --find-interpreter \ - --manifest-path pineappl_cli/Cargo.toml + --target $TARGET --manifest-path pineappl_cli/Cargo.toml - name: Patch wheels run: | delocate-wheel dist/* From 62a2e34197b61dadd12d523e6b7651d04304e39f Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 12:32:13 +0400 Subject: [PATCH 23/92] Use cibuildwheel on Linux, disable Windows --- .github/workflows/cli-wheels.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index b8081f0e7..e81a54bfc 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -16,7 +16,6 @@ env: jobs: wheels-linux: runs-on: ubuntu-latest - container: quay.io/pypa/manylinux2014_x86_64 strategy: matrix: target: [x86_64] @@ -41,19 +40,14 @@ jobs: architecture: ${{ matrix.target }} - name: Install build tools run: | - pip install maturin auditwheel + pip install cibuildwheel - name: Install non-Rust dependencies run: | python maintainer/install-lhapdf.py - name: Build wheels + working-directory: pineappl_cli run: | - # `--find-interpreter` is needed to generate wheels for - # *all* Python versions - maturin build --release --out dist --find-interpreter \ - --manifest-path pineappl_cli/Cargo.toml - - name: Patch wheels - run: | - auditwheel repair dist/* + cibuildwheel --out ../dist - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -121,6 +115,7 @@ jobs: strategy: matrix: target: [x64] + if: false steps: - uses: actions/checkout@v4 From 107ed5f20de6b6e19d4410220d520f4ca30d0cd8 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 12:48:21 +0400 Subject: [PATCH 24/92] Attempt building macos wheels on native runners --- .github/workflows/cli-wheels.yml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index e81a54bfc..dd904e6ab 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -55,10 +55,14 @@ jobs: path: dist wheels-macos: - runs-on: macos-latest + runs-on: ${{ matrix.os }} strategy: matrix: - target: [aarch64, x86_64] + include: + - os: macos-13 + target: x86_64 + - os: macos-14 + target: aarch64 steps: - uses: actions/checkout@v4 @@ -78,6 +82,7 @@ jobs: 3.11 3.12 3.10 + architecture: ${{ matrix.target }} - name: Install build tools run: | pip install maturin delocate @@ -86,20 +91,10 @@ jobs: python maintainer/install-lhapdf.py - name: Build wheels run: | - case ${{ matrix.target }} in - aarch64) - TARGET="aarch64-apple-darwin" - ;; - x86_64) - TARGET="x86_64-apple-darwin" - ;; - *) - exit 1 - esac # `--find-interpreter` is needed to generate wheels for # *all* Python versions maturin build --release --out dist --find-interpreter \ - --target $TARGET --manifest-path pineappl_cli/Cargo.toml + --manifest-path pineappl_cli/Cargo.toml - name: Patch wheels run: | delocate-wheel dist/* From 2da9b5ead2903c2a75ecd487d6debc2a04ed1e5d Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 12:49:19 +0400 Subject: [PATCH 25/92] Restore Node 20 version of actions on Linux --- .github/workflows/cli-wheels.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index dd904e6ab..cff2f0d3b 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -21,13 +21,14 @@ jobs: target: [x86_64] steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the # default one. Apparently maturin doesn't find all Python versions # when one chooses a PyPy version instead of CPython as default python-version: | + pypy3.7 pypy3.8 pypy3.9 pypy3.10 From c16acde67caf906d8dbcfc4512be3e8308e41df7 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 12:52:23 +0400 Subject: [PATCH 26/92] Drop architecture specification from setup python --- .github/workflows/cli-wheels.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index cff2f0d3b..bf65f54b8 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -38,7 +38,6 @@ jobs: 3.11 3.12 3.10 - architecture: ${{ matrix.target }} - name: Install build tools run: | pip install cibuildwheel @@ -83,7 +82,6 @@ jobs: 3.11 3.12 3.10 - architecture: ${{ matrix.target }} - name: Install build tools run: | pip install maturin delocate @@ -131,7 +129,6 @@ jobs: 3.11 3.12 3.10 - architecture: ${{ matrix.target }} - name: Install build tools run: | pip install maturin delvewheel From dc4a3760cea3682eb7d58e6f17fc2dd6b5a2641a Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 13:02:52 +0400 Subject: [PATCH 27/92] Skip building wheels for CPython 3.6 --- .github/workflows/cli-wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index bf65f54b8..a1a4fddce 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -47,6 +47,7 @@ jobs: - name: Build wheels working-directory: pineappl_cli run: | + CIBW_SKIP="cp36-*" cibuildwheel --out ../dist - name: Upload wheels uses: actions/upload-artifact@v4 From 83222494414d080809705c8587900d114ccc2c30 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 13:04:29 +0400 Subject: [PATCH 28/92] Skip PyPy 3.7 on macos --- .github/workflows/cli-wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index a1a4fddce..8494068a7 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -73,7 +73,6 @@ jobs: # default one. Apparently maturin doesn't find all Python versions # when one chooses a PyPy version instead of CPython as default python-version: | - pypy3.7 pypy3.8 pypy3.9 pypy3.10 From 6101d5d5215b67b8814a13063076b185de158bd6 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 13:06:47 +0400 Subject: [PATCH 29/92] Skip CPython 3.7 as well --- .github/workflows/cli-wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 8494068a7..0ed05d45c 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -76,7 +76,6 @@ jobs: pypy3.8 pypy3.9 pypy3.10 - 3.7 3.8 3.9 3.11 From 43f601a666fa832728457de897ecd50965c8e748 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 13:07:33 +0400 Subject: [PATCH 30/92] Attempt exporting env var for cibuildwheel --- .github/workflows/cli-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 0ed05d45c..a992aed92 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -47,7 +47,7 @@ jobs: - name: Build wheels working-directory: pineappl_cli run: | - CIBW_SKIP="cp36-*" + export CIBW_SKIP="cp36-*" cibuildwheel --out ../dist - name: Upload wheels uses: actions/upload-artifact@v4 From c449f1e6512d5689eb682c9edfac25ca4c19fedd Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 13:11:08 +0400 Subject: [PATCH 31/92] Fix cibuildwheel invocation, drop one Python 3.8 as well on macos --- .github/workflows/cli-wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index a992aed92..15e540522 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -48,7 +48,7 @@ jobs: working-directory: pineappl_cli run: | export CIBW_SKIP="cp36-*" - cibuildwheel --out ../dist + cibuildwheel --output-dir ../dist . - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -76,7 +76,6 @@ jobs: pypy3.8 pypy3.9 pypy3.10 - 3.8 3.9 3.11 3.12 From c19ef0dd521e89bc290471f4dc5729944b958960 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 13:18:18 +0400 Subject: [PATCH 32/92] Install Rust toolchain on Linux --- .github/workflows/cli-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 15e540522..a631a9f1e 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -22,6 +22,7 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the @@ -76,7 +77,6 @@ jobs: pypy3.8 pypy3.9 pypy3.10 - 3.9 3.11 3.12 3.10 From 66cc22ad42e3dde1f1671b4472af91742acbc9d7 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 22:36:29 +0400 Subject: [PATCH 33/92] Restore maturin action on linux --- .github/workflows/cli-wheels.yml | 38 +++++++++----------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index a631a9f1e..1bc459084 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -22,34 +22,16 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v5 - with: - # WARNING: be careful with the ordering - the last version is the - # default one. Apparently maturin doesn't find all Python versions - # when one chooses a PyPy version instead of CPython as default - python-version: | - pypy3.7 - pypy3.8 - pypy3.9 - pypy3.10 - 3.7 - 3.8 - 3.9 - 3.11 - 3.12 - 3.10 - - name: Install build tools - run: | - pip install cibuildwheel - - name: Install non-Rust dependencies - run: | - python maintainer/install-lhapdf.py - name: Build wheels - working-directory: pineappl_cli - run: | - export CIBW_SKIP="cp36-*" - cibuildwheel --output-dir ../dist . + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + before-script-linux: | + python maintainer/install-lhapdf.py + # `--find-interpreter` is needed to generate wheels for *all* Python versions + args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml + sccache: "true" + manylinux: auto - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -85,6 +67,8 @@ jobs: pip install maturin delocate - name: Install non-Rust dependencies run: | + brew update + brew install autoconf automake libtool python maintainer/install-lhapdf.py - name: Build wheels run: | From 392cbfd26f9d88fcfd3ce0d7f29054fe503a3791 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Tue, 26 Mar 2024 23:20:54 +0400 Subject: [PATCH 34/92] Temporary ignore macos arm --- .github/workflows/cli-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 1bc459084..eff8e3ee3 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -45,8 +45,8 @@ jobs: include: - os: macos-13 target: x86_64 - - os: macos-14 - target: aarch64 + # - os: macos-14 + # target: aarch64 steps: - uses: actions/checkout@v4 From 70c8c36fbb873d1be1c648647a02ec5a527c273e Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 07:11:28 +0400 Subject: [PATCH 35/92] Back to maturin action for macos --- .github/workflows/cli-wheels.yml | 22 +++++++++------------- maintainer/patch-macos-wheel.py | 2 +- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index eff8e3ee3..342b33e20 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -39,14 +39,10 @@ jobs: path: dist wheels-macos: - runs-on: ${{ matrix.os }} + runs-on: macos-latest strategy: matrix: - include: - - os: macos-13 - target: x86_64 - # - os: macos-14 - # target: aarch64 + target: [aarch64, x86_64] steps: - uses: actions/checkout@v4 @@ -64,22 +60,22 @@ jobs: 3.10 - name: Install build tools run: | - pip install maturin delocate + pip install delocate - name: Install non-Rust dependencies run: | brew update brew install autoconf automake libtool python maintainer/install-lhapdf.py - name: Build wheels - run: | - # `--find-interpreter` is needed to generate wheels for - # *all* Python versions - maturin build --release --out dist --find-interpreter \ - --manifest-path pineappl_cli/Cargo.toml + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml + sccache: "true" - name: Patch wheels run: | delocate-wheel dist/* - python maintainer/patch-macos-wheel.py + python maintainer/patch-macos-wheel.py dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: diff --git a/maintainer/patch-macos-wheel.py b/maintainer/patch-macos-wheel.py index f00be39ba..8d9b62d52 100644 --- a/maintainer/patch-macos-wheel.py +++ b/maintainer/patch-macos-wheel.py @@ -71,7 +71,7 @@ def patch(wheel: Path): installed.write_text(WRAPPER.format(binary_path=str(binary.relative_to(tmpd)))) next(Path(tmpd).glob("**/RECORD")).write_text(record(tmpd)) shutil.make_archive(wheel.name, format="zip", base_dir=".", root_dir=tmpd) - shutil.move(str(wheel) + ".zip", wheel) + shutil.move(wheel.name + ".zip", wheel) log.info(f"Patched {wheel} in-place") From e135b0c778eff5e440dae64a1431cb45b984db4c Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 07:16:20 +0400 Subject: [PATCH 36/92] Ignore macos aarch64 once more --- .github/workflows/cli-wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 342b33e20..2356ee151 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -42,7 +42,8 @@ jobs: runs-on: macos-latest strategy: matrix: - target: [aarch64, x86_64] + # target: [aarch64, x86_64] + target: [x86_64] steps: - uses: actions/checkout@v4 From f1d080d26053e2b8cc20d43d2cc6bf49fc198034 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 07:16:53 +0400 Subject: [PATCH 37/92] Disambiguate python major in command --- .github/workflows/cli-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 2356ee151..29bba86f5 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -27,7 +27,7 @@ jobs: with: target: ${{ matrix.target }} before-script-linux: | - python maintainer/install-lhapdf.py + python3 maintainer/install-lhapdf.py # `--find-interpreter` is needed to generate wheels for *all* Python versions args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml sccache: "true" From 33165a104ff1ff8e204dc96fc2eebfd819f63090 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 07:33:56 +0400 Subject: [PATCH 38/92] Restore python versions and target --- .github/workflows/cli-wheels.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 29bba86f5..da8fa3cad 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -41,9 +41,9 @@ jobs: wheels-macos: runs-on: macos-latest strategy: + fail-fast: false matrix: - # target: [aarch64, x86_64] - target: [x86_64] + target: [aarch64, x86_64] steps: - uses: actions/checkout@v4 @@ -53,9 +53,13 @@ jobs: # default one. Apparently maturin doesn't find all Python versions # when one chooses a PyPy version instead of CPython as default python-version: | + pypy3.7 pypy3.8 pypy3.9 pypy3.10 + 3.7 + 3.8 + 3.9 3.11 3.12 3.10 From 84172439b88eee33ee6841904d62a4c5bfec3050 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 07:38:21 +0400 Subject: [PATCH 39/92] Remove again some Python 3.7 --- .github/workflows/cli-wheels.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index da8fa3cad..3a46d26f7 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -53,11 +53,9 @@ jobs: # default one. Apparently maturin doesn't find all Python versions # when one chooses a PyPy version instead of CPython as default python-version: | - pypy3.7 pypy3.8 pypy3.9 pypy3.10 - 3.7 3.8 3.9 3.11 From 326fa2b84045a0652dd08cade851ef8e90594e7c Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 07:47:26 +0400 Subject: [PATCH 40/92] Install g++ in manylinux containers --- .github/workflows/cli-wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 3a46d26f7..852bbdc5d 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -27,6 +27,7 @@ jobs: with: target: ${{ matrix.target }} before-script-linux: | + yum -y install gcc-c++ python3 maintainer/install-lhapdf.py # `--find-interpreter` is needed to generate wheels for *all* Python versions args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml From 2ad020eaab9ca463123f5b62c1ad08f8bbab188c Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 08:09:01 +0400 Subject: [PATCH 41/92] Add even more packages to manylinux container --- .github/workflows/cli-wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 852bbdc5d..aa5e70507 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -27,7 +27,8 @@ jobs: with: target: ${{ matrix.target }} before-script-linux: | - yum -y install gcc-c++ + yum update + yum -y install gcc-c++ make python3 maintainer/install-lhapdf.py # `--find-interpreter` is needed to generate wheels for *all* Python versions args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml From 3855495668174b2ed9edcdf30318321080630b40 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 09:33:03 +0400 Subject: [PATCH 42/92] Unlock windows wheels --- .github/workflows/cli-wheels.yml | 19 ++++++++++--------- ...h-macos-wheel.py => patch-binary-wheel.py} | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) rename maintainer/{patch-macos-wheel.py => patch-binary-wheel.py} (95%) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index aa5e70507..e4d3e7b46 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -80,7 +80,7 @@ jobs: - name: Patch wheels run: | delocate-wheel dist/* - python maintainer/patch-macos-wheel.py dist/* + python maintainer/patch-binary-wheel.py dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -92,7 +92,7 @@ jobs: strategy: matrix: target: [x64] - if: false + # if: false steps: - uses: actions/checkout@v4 @@ -112,18 +112,19 @@ jobs: 3.11 3.12 3.10 - - name: Install build tools - run: | - pip install maturin delvewheel - name: Install non-Rust dependencies run: | python maintainer/install-lhapdf.py - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml + sccache: "true" + - name: Patch wheels run: | - # `--find-interpreter` is needed to generate wheels for - # *all* Python versions - maturin build --release --out dist --find-interpreter \ - --manifest-path pineappl_cli/Cargo.toml + delvewheel repair dist/* + python maintainer/patch-binary-wheel.py dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: diff --git a/maintainer/patch-macos-wheel.py b/maintainer/patch-binary-wheel.py similarity index 95% rename from maintainer/patch-macos-wheel.py rename to maintainer/patch-binary-wheel.py index 8d9b62d52..62dcfbf39 100644 --- a/maintainer/patch-macos-wheel.py +++ b/maintainer/patch-binary-wheel.py @@ -1,4 +1,4 @@ -"""Patch MacOS wheel to allow executable binaries. +"""Patch binary wheel to allow executable binaries. Following the patch in https://github.com/pypa/auditwheel/pull/443/ """ @@ -12,7 +12,7 @@ from pathlib import Path logging.basicConfig() -log = logging.getLogger("Patch MacOS wheels") +log = logging.getLogger("binary-patch") log.setLevel(logging.INFO) WRAPPER = """ From 97a6bb93e269c4d51bef5cde00f98ad75581a2be Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 09:45:07 +0400 Subject: [PATCH 43/92] Install cygwin for windows LHAPDF compilation --- .github/workflows/cli-wheels.yml | 1 + maintainer/.gitignore | 1 + maintainer/{install-lhapdf.py => install_lhapdf.py} | 8 +++++--- .../{patch-binary-wheel.py => patch_binary_wheel.py} | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 maintainer/.gitignore rename maintainer/{install-lhapdf.py => install_lhapdf.py} (89%) rename maintainer/{patch-binary-wheel.py => patch_binary_wheel.py} (99%) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index e4d3e7b46..5b8d15ee6 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -114,6 +114,7 @@ jobs: 3.10 - name: Install non-Rust dependencies run: | + choco install cygwin python maintainer/install-lhapdf.py - name: Build wheels uses: PyO3/maturin-action@v1 diff --git a/maintainer/.gitignore b/maintainer/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/maintainer/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/maintainer/install-lhapdf.py b/maintainer/install_lhapdf.py similarity index 89% rename from maintainer/install-lhapdf.py rename to maintainer/install_lhapdf.py index 66e13545a..08c79688a 100644 --- a/maintainer/install-lhapdf.py +++ b/maintainer/install_lhapdf.py @@ -37,14 +37,16 @@ def run_(cmd, *args, **kwargs): cmd = cmd.split() if isinstance(cmd, str) else cmd run(cmd, *args, cwd=path / SRC_DIR, **kwargs) + log.info("Autoreconf...") run_("autoreconf -f -i") prefix = os.environ.get("PREFIX") prefix_ = ["--prefix", prefix] if prefix is not None else [] + log.info("Configuring...") run_(["./configure"] + prefix_, env={"PYTHON": sys.executable}) - log.info("Configured") + log.info("Building...") run_("make clean") run_("make -j") - log.info("Built") + log.info("Installing...") run_("make install") log.info("Installed") @@ -55,7 +57,7 @@ def main(): log.info(f"Downloaded {HOST}{PATH}") with tempfile.TemporaryDirectory() as tmpd: tarfile.open(fileobj=io.BytesIO(tar), mode="r:gz").extractall(tmpd) - log.info(f"Extracted LHAPDF tarbal in {tmpd}") + log.info(f"Extracted LHAPDF tarball in {tmpd}") install(Path(tmpd)) diff --git a/maintainer/patch-binary-wheel.py b/maintainer/patch_binary_wheel.py similarity index 99% rename from maintainer/patch-binary-wheel.py rename to maintainer/patch_binary_wheel.py index 62dcfbf39..00da4be6b 100644 --- a/maintainer/patch-binary-wheel.py +++ b/maintainer/patch_binary_wheel.py @@ -15,11 +15,12 @@ log = logging.getLogger("binary-patch") log.setLevel(logging.INFO) -WRAPPER = """ +WRAPPER = """\ #!python import os import sys import sysconfig + if __name__ == "__main__": os.execv( os.path.join(sysconfig.get_path("platlib"), {binary_path!r}), From c4fa8e0b79584f41ba7923349cea952f41842652 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 09:47:59 +0400 Subject: [PATCH 44/92] Propagate scripts' names fix --- .github/workflows/cli-wheels.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 5b8d15ee6..c03b3351d 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -29,7 +29,7 @@ jobs: before-script-linux: | yum update yum -y install gcc-c++ make - python3 maintainer/install-lhapdf.py + python3 maintainer/install_lhapdf.py # `--find-interpreter` is needed to generate wheels for *all* Python versions args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml sccache: "true" @@ -70,7 +70,7 @@ jobs: run: | brew update brew install autoconf automake libtool - python maintainer/install-lhapdf.py + python maintainer/install_lhapdf.py - name: Build wheels uses: PyO3/maturin-action@v1 with: @@ -80,7 +80,7 @@ jobs: - name: Patch wheels run: | delocate-wheel dist/* - python maintainer/patch-binary-wheel.py dist/* + python maintainer/patch_binary_wheel.py dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -115,7 +115,7 @@ jobs: - name: Install non-Rust dependencies run: | choco install cygwin - python maintainer/install-lhapdf.py + python maintainer/install_lhapdf.py - name: Build wheels uses: PyO3/maturin-action@v1 with: @@ -125,7 +125,7 @@ jobs: - name: Patch wheels run: | delvewheel repair dist/* - python maintainer/patch-binary-wheel.py dist/* + python maintainer/patch_binary_wheel.py dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: From fa512a27cd55cf128bc0707254e11fda8fd08360 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 09:52:08 +0400 Subject: [PATCH 45/92] Avoid autoreconf during LHAPDF installation Currently failing on Windows, and not part of the official LHAPDF instructions --- maintainer/install_lhapdf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maintainer/install_lhapdf.py b/maintainer/install_lhapdf.py index 08c79688a..a394b2cee 100644 --- a/maintainer/install_lhapdf.py +++ b/maintainer/install_lhapdf.py @@ -37,8 +37,8 @@ def run_(cmd, *args, **kwargs): cmd = cmd.split() if isinstance(cmd, str) else cmd run(cmd, *args, cwd=path / SRC_DIR, **kwargs) - log.info("Autoreconf...") - run_("autoreconf -f -i") + # log.info("Autoreconf...") + # run_("autoreconf -f -i") prefix = os.environ.get("PREFIX") prefix_ = ["--prefix", prefix] if prefix is not None else [] log.info("Configuring...") From 1bd3d096ea26a43160b9a82013920336c35bf0db Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 09:55:45 +0400 Subject: [PATCH 46/92] Replace cygwin with msys2 --- .github/workflows/cli-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index c03b3351d..b17384152 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -114,7 +114,7 @@ jobs: 3.10 - name: Install non-Rust dependencies run: | - choco install cygwin + choco install msys2 python maintainer/install_lhapdf.py - name: Build wheels uses: PyO3/maturin-action@v1 From 6fc7a4f858ade1c32ee1bfd001a0285e37f0edea Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 10:04:11 +0400 Subject: [PATCH 47/92] Consistently use msys2 to run windows --- .github/workflows/cli-wheels.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index b17384152..4e9745b1e 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -6,10 +6,6 @@ on: # - "v[0-9]+*" workflow_dispatch: -defaults: - run: - shell: bash - env: GH_TOKEN: ${{ github.token }} @@ -92,10 +88,18 @@ jobs: strategy: matrix: target: [x64] - # if: false + # if: false defaults: + defaults: + run: + shell: msys2 {0} steps: - uses: actions/checkout@v4 + - uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: true + install: git mingw-w64-ucrt-x86_64-gcc autoconf - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the @@ -114,7 +118,6 @@ jobs: 3.10 - name: Install non-Rust dependencies run: | - choco install msys2 python maintainer/install_lhapdf.py - name: Build wheels uses: PyO3/maturin-action@v1 From 2b4c8fd98b28768d0b4b3d1f11318aa4436d8c77 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 10:07:40 +0400 Subject: [PATCH 48/92] Disambiguate python major in windows --- .github/workflows/cli-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 4e9745b1e..f75a34271 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -118,7 +118,7 @@ jobs: 3.10 - name: Install non-Rust dependencies run: | - python maintainer/install_lhapdf.py + python3 maintainer/install_lhapdf.py - name: Build wheels uses: PyO3/maturin-action@v1 with: @@ -128,7 +128,7 @@ jobs: - name: Patch wheels run: | delvewheel repair dist/* - python maintainer/patch_binary_wheel.py dist/* + python3 maintainer/patch_binary_wheel.py dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: From 4c06c8f64ab1e2d910a7d8c2cc613ab68dbcb0e4 Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 10:12:24 +0400 Subject: [PATCH 49/92] Explicitly add tools to msys2 --- .github/workflows/cli-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index f75a34271..943eb2105 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -99,7 +99,7 @@ jobs: with: msystem: UCRT64 update: true - install: git mingw-w64-ucrt-x86_64-gcc autoconf + install: git mingw-w64-ucrt-x86_64-gcc autoconf python3 - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the From b1cc3fb6c36de05558723e3f2d792a486a281ccf Mon Sep 17 00:00:00 2001 From: Alessandro Candido Date: Wed, 27 Mar 2024 10:17:18 +0400 Subject: [PATCH 50/92] Change make version in msys2 --- .github/workflows/cli-wheels.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 943eb2105..9d6466b2c 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -99,7 +99,12 @@ jobs: with: msystem: UCRT64 update: true - install: git mingw-w64-ucrt-x86_64-gcc autoconf python3 + install: >- + git + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-make + autoconf + python3 - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the From e771c69addf41e5341c763678617273feafc9def Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 08:50:45 +0200 Subject: [PATCH 51/92] Link binary statically --- pineappl_cli/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pineappl_cli/pyproject.toml b/pineappl_cli/pyproject.toml index 0988846e1..94965fa49 100644 --- a/pineappl_cli/pyproject.toml +++ b/pineappl_cli/pyproject.toml @@ -4,3 +4,4 @@ build-backend = "maturin" [tool.maturin] bindings = "bin" +features = ["static"] From b82b3f5245e49908796e95e6db14f4bcd13d0adb Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 09:01:32 +0200 Subject: [PATCH 52/92] Use a different container when building CLI on Linux --- .github/workflows/cli-wheels.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 9d6466b2c..c40e0caac 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -22,10 +22,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - before-script-linux: | - yum update - yum -y install gcc-c++ make - python3 maintainer/install_lhapdf.py + container: ghcr.io/nnpdf/pineappl-ci:latest # `--find-interpreter` is needed to generate wheels for *all* Python versions args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml sccache: "true" From 4534082593d3fc1fec6b321ee0988b99a7830641 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:03:02 +0200 Subject: [PATCH 53/92] Try fixing macOS CLI wheel generation --- .github/workflows/cli-wheels.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index c40e0caac..a5e113873 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -56,24 +56,16 @@ jobs: 3.11 3.12 3.10 - - name: Install build tools + - name: Install LHAPDF run: | - pip install delocate - - name: Install non-Rust dependencies - run: | - brew update - brew install autoconf automake libtool - python maintainer/install_lhapdf.py + brew tap davidchall/hep + brew install lhapdf - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml sccache: "true" - - name: Patch wheels - run: | - delocate-wheel dist/* - python maintainer/patch_binary_wheel.py dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: From 01601b6d2bec5357e031d93a22484a655fc52933 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:07:57 +0200 Subject: [PATCH 54/92] Do not cross-compile macOS targets --- .github/workflows/cli-wheels.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index a5e113873..b4f6875f8 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -36,10 +36,14 @@ jobs: wheels-macos: runs-on: macos-latest strategy: - fail-fast: false matrix: - target: [aarch64, x86_64] - + include: + - os: macos-13 + target: x86_64 + - os: macos-14 + # we cannot cross-compile, because we need also all dependencies for the specified target + target: aarch64 + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From 8283730aace59178788b5b74e6e4fbb5f23fecc6 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:11:13 +0200 Subject: [PATCH 55/92] Fix YAML error in `cli-wheels.yml` --- .github/workflows/cli-wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index b4f6875f8..cf8c920c3 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -34,7 +34,6 @@ jobs: path: dist wheels-macos: - runs-on: macos-latest strategy: matrix: include: From 63300562856153b838a7ef017148012515b88e1a Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:29:35 +0200 Subject: [PATCH 56/92] Print `config.log` to debug Windows build --- .github/workflows/cli-wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index cf8c920c3..d0d554ff2 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -115,7 +115,8 @@ jobs: 3.10 - name: Install non-Rust dependencies run: | - python3 maintainer/install_lhapdf.py + python3 maintainer/install_lhapdf.py || true + cat LHAPDF-6.5.3/config.log - name: Build wheels uses: PyO3/maturin-action@v1 with: From f3b532d8b9501592836c83a152faeb8248bbc84a Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:33:35 +0200 Subject: [PATCH 57/92] Move CLI building for Linux and macOS to Release workflow --- .github/workflows/cli-wheels.yml | 65 -------------------------------- .github/workflows/release.yml | 64 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 65 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index d0d554ff2..bee07acc8 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -10,71 +10,6 @@ env: GH_TOKEN: ${{ github.token }} jobs: - wheels-linux: - runs-on: ubuntu-latest - strategy: - matrix: - target: [x86_64] - - steps: - - uses: actions/checkout@v4 - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - container: ghcr.io/nnpdf/pineappl-ci:latest - # `--find-interpreter` is needed to generate wheels for *all* Python versions - args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml - sccache: "true" - manylinux: auto - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-${{ matrix.target }} - path: dist - - wheels-macos: - strategy: - matrix: - include: - - os: macos-13 - target: x86_64 - - os: macos-14 - # we cannot cross-compile, because we need also all dependencies for the specified target - target: aarch64 - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - # WARNING: be careful with the ordering - the last version is the - # default one. Apparently maturin doesn't find all Python versions - # when one chooses a PyPy version instead of CPython as default - python-version: | - pypy3.8 - pypy3.9 - pypy3.10 - 3.8 - 3.9 - 3.11 - 3.12 - 3.10 - - name: Install LHAPDF - run: | - brew tap davidchall/hep - brew install lhapdf - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml - sccache: "true" - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: ${{ github.job }}-${{ matrix.target }} - path: dist - wheels-windows: runs-on: windows-latest strategy: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8fa0463cd..799788ef1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,6 +93,48 @@ jobs: name: pineappl_cli-${{ matrix.target }} path: pineappl_cli-${{ matrix.target }}.tar.gz + cli-wheels-macos: + strategy: + matrix: + include: + - os: macos-13 + target: x86_64 + - os: macos-14 + # we cannot cross-compile, because we need also all dependencies for the specified target + target: aarch64 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + # WARNING: be careful with the ordering - the last version is the + # default one. Apparently maturin doesn't find all Python versions + # when one chooses a PyPy version instead of CPython as default + python-version: | + pypy3.8 + pypy3.9 + pypy3.10 + 3.8 + 3.9 + 3.11 + 3.12 + 3.10 + - name: Install LHAPDF + run: | + brew tap davidchall/hep + brew install lhapdf + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml + sccache: "true" + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-${{ matrix.target }} + path: dist + capi-linux: runs-on: ubuntu-latest container: ghcr.io/nnpdf/pineappl-ci:latest @@ -143,6 +185,28 @@ jobs: name: pineappl_cli-${{ matrix.target }} path: pineappl_cli-${{ matrix.target }}.tar.gz + cli-wheels-linux: + runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64] + steps: + - uses: actions/checkout@v4 + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + container: ghcr.io/nnpdf/pineappl-ci:latest + # `--find-interpreter` is needed to generate wheels for *all* Python versions + args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml + sccache: "true" + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-${{ matrix.target }} + path: dist + # publish a release on github and upload pre-built CAPI publish-release: needs: [capi-macos, cli-macos, capi-linux, cli-linux] From 9b9b8b68fd0518c063566a8889354afe3340b961 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:35:20 +0200 Subject: [PATCH 58/92] Debug LHAPDF installation on Windows --- .github/workflows/cli-wheels.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index bee07acc8..9492077d0 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -32,6 +32,8 @@ jobs: mingw-w64-ucrt-x86_64-make autoconf python3 + curl + tar - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the @@ -48,10 +50,12 @@ jobs: 3.11 3.12 3.10 - - name: Install non-Rust dependencies + - name: Install LHAPDF run: | - python3 maintainer/install_lhapdf.py || true - cat LHAPDF-6.5.3/config.log + curl "https://lhapdf.hepforge.org/downloads/?f=LHAPDF-6.5.4.tar.gz" | tar xzf - + cd LHAPDF-6.5.4 + ./configure || true + cat LHAPDF-6.5.4/config.log - name: Build wheels uses: PyO3/maturin-action@v1 with: From 0cf77acdb67839af74c727d5c9e341314b693f09 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:37:34 +0200 Subject: [PATCH 59/92] Use same `actions/upload-artifact` version everywhere --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 799788ef1..9c7f4c545 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -130,7 +130,7 @@ jobs: args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml sccache: "true" - name: Upload wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: ${{ github.job }}-${{ matrix.target }} path: dist @@ -202,7 +202,7 @@ jobs: sccache: "true" manylinux: auto - name: Upload wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: ${{ github.job }}-${{ matrix.target }} path: dist From 68af11dafbb5d60858e10b6a6e7f3cecbf3175b7 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:42:49 +0200 Subject: [PATCH 60/92] Disable workflows temporarily --- .github/workflows/capi.yaml | 2 ++ .github/workflows/msrv.yml | 2 ++ .github/workflows/python.yml | 2 ++ .github/workflows/rust.yml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/.github/workflows/capi.yaml b/.github/workflows/capi.yaml index 4d15a0ad7..1d70c36bd 100644 --- a/.github/workflows/capi.yaml +++ b/.github/workflows/capi.yaml @@ -2,6 +2,8 @@ name: CAPI on: push: + branches-ignore: + - pycli jobs: capi: diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index 25ef4b9a0..c236f28ab 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -2,6 +2,8 @@ name: MSRV on: push: + branches-ignore: + - pycli env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index c9ccd8c8a..51d2a2568 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -2,6 +2,8 @@ name: Python on: push: + branches-ignore: + - pycli jobs: test: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a2751695b..17556a0a0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -2,6 +2,8 @@ name: Rust on: push: + branches-ignore: + - pycli defaults: run: From 16058295e8747b33109b2265d640fe423e7363d7 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:44:18 +0200 Subject: [PATCH 61/92] Fix printing `config.log` --- .github/workflows/cli-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 9492077d0..31695ddcd 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -55,7 +55,7 @@ jobs: curl "https://lhapdf.hepforge.org/downloads/?f=LHAPDF-6.5.4.tar.gz" | tar xzf - cd LHAPDF-6.5.4 ./configure || true - cat LHAPDF-6.5.4/config.log + cat config.log - name: Build wheels uses: PyO3/maturin-action@v1 with: From aa6e70f15dc41c2a5029cd3fa6834873a66ed910 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:48:58 +0200 Subject: [PATCH 62/92] Disable LHAPDF Python CLI --- .github/workflows/cli-wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 31695ddcd..09a24f23a 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -54,7 +54,8 @@ jobs: run: | curl "https://lhapdf.hepforge.org/downloads/?f=LHAPDF-6.5.4.tar.gz" | tar xzf - cd LHAPDF-6.5.4 - ./configure || true + # compiling with Python gives errors, but we also don't need Python + ./configure --disable-python || true cat config.log - name: Build wheels uses: PyO3/maturin-action@v1 From d343eb2f9ed650bca42a346e4061ec17dd450d34 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:53:37 +0200 Subject: [PATCH 63/92] Use standard gcc and make --- .github/workflows/cli-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 09a24f23a..de7b7f995 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -28,8 +28,8 @@ jobs: update: true install: >- git - mingw-w64-ucrt-x86_64-gcc - mingw-w64-ucrt-x86_64-make + gcc + make autoconf python3 curl From 3087388c1d74eb3aa893542a8e0f71528be2c555 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:55:49 +0200 Subject: [PATCH 64/92] Add `pkgconf` dependency --- .github/workflows/cli-wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index de7b7f995..dfbede633 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -34,6 +34,7 @@ jobs: python3 curl tar + pkgconf - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the From d591b4b48e7614e916834d4468de82603d54085a Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 10:59:28 +0200 Subject: [PATCH 65/92] Make and install LHAPDF --- .github/workflows/cli-wheels.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index dfbede633..f1ebbcbc0 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -56,8 +56,9 @@ jobs: curl "https://lhapdf.hepforge.org/downloads/?f=LHAPDF-6.5.4.tar.gz" | tar xzf - cd LHAPDF-6.5.4 # compiling with Python gives errors, but we also don't need Python - ./configure --disable-python || true - cat config.log + ./configure --disable-python + make -j + make install - name: Build wheels uses: PyO3/maturin-action@v1 with: From d359373434f3ff5994bc889a8185a74dcad8676c Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 11:13:29 +0200 Subject: [PATCH 66/92] Print path of `pkgconf` --- .github/workflows/cli-wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index f1ebbcbc0..4028e7544 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -59,6 +59,8 @@ jobs: ./configure --disable-python make -j make install + # print the path of `pkgconf` + command -v pkgconf - name: Build wheels uses: PyO3/maturin-action@v1 with: From 545e76815fa9462fd2d6a921cc08f7d65a4beae6 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 11:19:27 +0200 Subject: [PATCH 67/92] Try using `pkgconfig` instead of `pkg-config` --- .github/workflows/cli-wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 4028e7544..f392f594d 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -63,6 +63,8 @@ jobs: command -v pkgconf - name: Build wheels uses: PyO3/maturin-action@v1 + env: + PKG_CONFIG: "/usr/bin/pkgconf" with: target: ${{ matrix.target }} args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml From c2a41f41cd748050ddf23fec8d0f83e159ec8521 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 11:29:16 +0200 Subject: [PATCH 68/92] Use `pkg-config` instead `pkgconf` --- .github/workflows/cli-wheels.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index f392f594d..e74636dc4 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -34,7 +34,7 @@ jobs: python3 curl tar - pkgconf + mingw-w64-pkg-config - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the @@ -60,11 +60,9 @@ jobs: make -j make install # print the path of `pkgconf` - command -v pkgconf + command -v pkg-config - name: Build wheels uses: PyO3/maturin-action@v1 - env: - PKG_CONFIG: "/usr/bin/pkgconf" with: target: ${{ matrix.target }} args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml From d952c62bf133fa2205caed7a1e035cfc864902ea Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 11:34:31 +0200 Subject: [PATCH 69/92] Try a different name for `pkg-config` --- .github/workflows/cli-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index e74636dc4..aedcfede4 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -34,7 +34,7 @@ jobs: python3 curl tar - mingw-w64-pkg-config + mingw-w64-ucrt-x86_64-pkg-config - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the From 621cec47c29463705c5fcfcd49b8e09194a56690 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 11:45:04 +0200 Subject: [PATCH 70/92] Set the path to `pkg-config` explicitly --- .github/workflows/cli-wheels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index aedcfede4..043679df5 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -61,6 +61,10 @@ jobs: make install # print the path of `pkgconf` command -v pkg-config + - name: Set path to pkg-config + run: | + # for some reason `pkg-config` isn't properly found by Rust + echo "PKG_CONFIG=$(command -v pkg-config)" >> ${GITHUB_ENV} - name: Build wheels uses: PyO3/maturin-action@v1 with: From 228c34d4c4c051172bddb90ea5fa3b9b61aa456e Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 11:55:27 +0200 Subject: [PATCH 71/92] Disable shared library generation for LHAPDF --- .github/workflows/cli-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 043679df5..697b8314a 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -56,7 +56,7 @@ jobs: curl "https://lhapdf.hepforge.org/downloads/?f=LHAPDF-6.5.4.tar.gz" | tar xzf - cd LHAPDF-6.5.4 # compiling with Python gives errors, but we also don't need Python - ./configure --disable-python + ./configure --disable-python --disable-shared make -j make install # print the path of `pkgconf` From dde69ae070dffb710db03e23625043f7a006ff32 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 11:58:01 +0200 Subject: [PATCH 72/92] Debug `pkg-config` paths --- .github/workflows/cli-wheels.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 697b8314a..64c9f3deb 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -61,10 +61,12 @@ jobs: make install # print the path of `pkgconf` command -v pkg-config - - name: Set path to pkg-config + - name: Set pkg-config paths run: | # for some reason `pkg-config` isn't properly found by Rust echo "PKG_CONFIG=$(command -v pkg-config)" >> ${GITHUB_ENV} + echo "PKG_CONFIG_PATH=/ucrt64/lib/pkgconfig:${PKG_CONFIG_PATH}" >> ${GITHUB_ENV} + echo "${PKG_CONFIG_PATH}" - name: Build wheels uses: PyO3/maturin-action@v1 with: From 371ffe527de9baa5704c6c20b0c0c3a3356d2e46 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 15:15:23 +0200 Subject: [PATCH 73/92] Rename jobs --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9c7f4c545..f3b560793 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,7 +93,7 @@ jobs: name: pineappl_cli-${{ matrix.target }} path: pineappl_cli-${{ matrix.target }}.tar.gz - cli-wheels-macos: + cli-wheel-macos: strategy: matrix: include: @@ -185,7 +185,7 @@ jobs: name: pineappl_cli-${{ matrix.target }} path: pineappl_cli-${{ matrix.target }}.tar.gz - cli-wheels-linux: + cli-wheel-linux: runs-on: ubuntu-latest strategy: matrix: From 11fd8b42ae8e2b7deadf9facf1f03e4bd1645d62 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 15:19:36 +0200 Subject: [PATCH 74/92] Activate converters on Linux --- .github/workflows/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f3b560793..24c09f1f0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -192,6 +192,9 @@ jobs: target: [x86_64] steps: - uses: actions/checkout@v4 + - name: Enable APPLgrid, fastNLO and legacy FK-table converters + run: | + sed -i "s/features = [\"static\"]/all-features = true/" pineappl_cli/pyproject.toml - name: Build wheels uses: PyO3/maturin-action@v1 with: From b742558a4fc01d3989b069b1d51a420136b3e2de Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 15:31:01 +0200 Subject: [PATCH 75/92] Build CLI with all features enabled --- .github/workflows/release.yml | 3 --- pineappl_cli/pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 24c09f1f0..f3b560793 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -192,9 +192,6 @@ jobs: target: [x86_64] steps: - uses: actions/checkout@v4 - - name: Enable APPLgrid, fastNLO and legacy FK-table converters - run: | - sed -i "s/features = [\"static\"]/all-features = true/" pineappl_cli/pyproject.toml - name: Build wheels uses: PyO3/maturin-action@v1 with: diff --git a/pineappl_cli/pyproject.toml b/pineappl_cli/pyproject.toml index 94965fa49..a5476cc3d 100644 --- a/pineappl_cli/pyproject.toml +++ b/pineappl_cli/pyproject.toml @@ -4,4 +4,4 @@ build-backend = "maturin" [tool.maturin] bindings = "bin" -features = ["static"] +all-features = true From 70ac438b00240bc9074265e9595edf3f56a6b85c Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 15:31:20 +0200 Subject: [PATCH 76/92] Install CLI dependencies on macOS --- .github/workflows/release.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f3b560793..dea8c2a52 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,10 +119,28 @@ jobs: 3.11 3.12 3.10 - - name: Install LHAPDF + - name: Install dependencies run: | brew tap davidchall/hep + # install LHAPDF brew install lhapdf + # install APPLgrid's dependencies; disable ROOT because its static libraries are missing + brew install --only-dependencies --without-hoppet --without-lhapdf --without-root applgrid + # install zlib, which is a dependency of APPLgrid but somehow missing + brew install zlib + # APPLgrid wants to be linked against zlib, and we need to find its static library via pkg-config + export PKG_CONFIG_PATH=$(find $(brew --cellar) -name '*.pc' -exec dirname {} + | sort -u | tr '\n' ':') + # manually compile APPLgrid, because we need the file `appl_igrid.h` and the files it includes, which are possibly generated + export HOMEBREW_TEMP="$(pwd)"/tmp + mkdir -p ${HOMEBREW_TEMP} + brew install --build-from-source --keep-tmp --without-hoppet --without-lhapdf --without-root applgrid + export APPL_IGRID_DIR=$(find ${HOMEBREW_TEMP} -name appl_igrid.h -exec dirname {} +) + # do not link against `gfortran` + sed 's/-lgfortran//g' $(command -v applgrid-config) > applgrid-config.new + mv applgrid-config.new $(command -v applgrid-config) + chmod +x $(command -v applgrid-config) + # install fastNLO + brew install --without-fastjet fastnlo - name: Build wheels uses: PyO3/maturin-action@v1 with: From 9b4394cfd70e8d0392eaca821f17adcfffbe79da Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 15:39:15 +0200 Subject: [PATCH 77/92] Export environment variables properly --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dea8c2a52..ad10cdffd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -129,12 +129,12 @@ jobs: # install zlib, which is a dependency of APPLgrid but somehow missing brew install zlib # APPLgrid wants to be linked against zlib, and we need to find its static library via pkg-config - export PKG_CONFIG_PATH=$(find $(brew --cellar) -name '*.pc' -exec dirname {} + | sort -u | tr '\n' ':') + echo "PKG_CONFIG_PATH=$(find $(brew --cellar) -name '*.pc' -exec dirname {} + | sort -u | tr '\n' ':')" >> ${GITHUB_ENV} # manually compile APPLgrid, because we need the file `appl_igrid.h` and the files it includes, which are possibly generated export HOMEBREW_TEMP="$(pwd)"/tmp mkdir -p ${HOMEBREW_TEMP} brew install --build-from-source --keep-tmp --without-hoppet --without-lhapdf --without-root applgrid - export APPL_IGRID_DIR=$(find ${HOMEBREW_TEMP} -name appl_igrid.h -exec dirname {} +) + echo "APPL_IGRID_DIR=$(find ${HOMEBREW_TEMP} -name appl_igrid.h -exec dirname {} +)" >> ${GITHUB_ENV} # do not link against `gfortran` sed 's/-lgfortran//g' $(command -v applgrid-config) > applgrid-config.new mv applgrid-config.new $(command -v applgrid-config) From 36ab312de31ccd79a3a8b3a6e437e5fe595eef07 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 16:01:01 +0200 Subject: [PATCH 78/92] Add cosmetic changes --- .github/workflows/release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad10cdffd..8b77142bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: workflow_dispatch: env: + # this is make the `gh` binary work GH_TOKEN: ${{ github.token }} jobs: @@ -93,7 +94,7 @@ jobs: name: pineappl_cli-${{ matrix.target }} path: pineappl_cli-${{ matrix.target }}.tar.gz - cli-wheel-macos: + cli-wheels-macos: strategy: matrix: include: @@ -203,7 +204,7 @@ jobs: name: pineappl_cli-${{ matrix.target }} path: pineappl_cli-${{ matrix.target }}.tar.gz - cli-wheel-linux: + cli-wheels-linux: runs-on: ubuntu-latest strategy: matrix: From c29b0567038fe1fcf716e7ac9aaf1107d1ad2f11 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 19:07:23 +0200 Subject: [PATCH 79/92] Try running `maturin` manually --- .github/workflows/cli-wheels.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 64c9f3deb..e2c297d47 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -35,6 +35,8 @@ jobs: curl tar mingw-w64-ucrt-x86_64-pkg-config + rust + maturin - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the @@ -68,11 +70,8 @@ jobs: echo "PKG_CONFIG_PATH=/ucrt64/lib/pkgconfig:${PKG_CONFIG_PATH}" >> ${GITHUB_ENV} echo "${PKG_CONFIG_PATH}" - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml - sccache: "true" + run: + maturin build --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml --target ${{ matrix.target }} - name: Patch wheels run: | delvewheel repair dist/* From f1336ca3148eb99c2b86a20f718ac7ac09bae543 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 19:08:59 +0200 Subject: [PATCH 80/92] Remove binary patching --- .github/workflows/cli-wheels.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index e2c297d47..4326b9d9e 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -72,10 +72,6 @@ jobs: - name: Build wheels run: maturin build --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml --target ${{ matrix.target }} - - name: Patch wheels - run: | - delvewheel repair dist/* - python3 maintainer/patch_binary_wheel.py dist/* - name: Upload wheels uses: actions/upload-artifact@v4 with: From cab2cb973a300b2075382445a14f4ad7d5352474 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Fri, 28 Jun 2024 19:11:20 +0200 Subject: [PATCH 81/92] Correct names of Rust and maturin packages --- .github/workflows/cli-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels.yml index 4326b9d9e..3f6dcf361 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels.yml @@ -35,8 +35,8 @@ jobs: curl tar mingw-w64-ucrt-x86_64-pkg-config - rust - maturin + mingw-w64-rust + mingw-w64-python-maturin - uses: actions/setup-python@v5 with: # WARNING: be careful with the ordering - the last version is the From fb5b4dc623eb58a4baa3eddc7be81259f6dc3188 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 09:33:01 +0200 Subject: [PATCH 82/92] Remove obsolete/unrelated files --- .envrc | 11 - .gitignore | 4 - docs/maintainers-guide.md | 7 - flake.lock | 376 ------------------------------- flake.nix | 70 ------ maintainer/.gitignore | 1 - maintainer/install_lhapdf.py | 65 ------ maintainer/patch_binary_wheel.py | 88 -------- 8 files changed, 622 deletions(-) delete mode 100644 .envrc delete mode 100644 flake.lock delete mode 100644 flake.nix delete mode 100644 maintainer/.gitignore delete mode 100644 maintainer/install_lhapdf.py delete mode 100644 maintainer/patch_binary_wheel.py diff --git a/.envrc b/.envrc deleted file mode 100644 index b351dff3d..000000000 --- a/.envrc +++ /dev/null @@ -1,11 +0,0 @@ -if ! has nix_direnv_version || ! nix_direnv_version 2.2.1; then - source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.2.1/direnvrc" "sha256-zelF0vLbEl5uaqrfIzbgNzJWGmLzCmYAkInj/LNxvKs=" -fi - -nix_direnv_watch_file devenv.nix -nix_direnv_watch_file devenv.lock -nix_direnv_watch_file devenv.yaml -if ! use flake . --impure -then - echo "devenv could not be built. The devenv environment was not loaded. Make the necessary changes to devenv.nix and hit enter to try again." >&2 -fi diff --git a/.gitignore b/.gitignore index 2b1938dbb..5af45873c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,2 @@ /target - -# Devenv files, to handle the development shell defined by the flake -.devenv/ - test-data/* diff --git a/docs/maintainers-guide.md b/docs/maintainers-guide.md index e7e5cddeb..00d857baa 100644 --- a/docs/maintainers-guide.md +++ b/docs/maintainers-guide.md @@ -35,7 +35,6 @@ - `pineappl_py`: the crate that builds PineAPPL's Python interface, documentation at - `xtask`: crate for [cargo-xtask] commands -- `.envrc`: [direnv] hooks (mainly enable Nix shell) - `.gitignore`: PineAPPL's Git ignore rules - `.readthedocs.yml`: configuration for PineAPPL's [Read-the-Docs] Python interface documentation @@ -47,9 +46,6 @@ - `README.md`: PineAPPL's repository README file. This file also serves as the homepage of [PineAPPL's website] - `_config.yml`: configuration file for PineAPPL's Github-pages website -- `flake.lock`: Nix [flake lock file] -- `flake.nix`: Nix [flake], which tracks cross-language dependencies to define a - reproducible build for the PineAPPL packages, and a suitable development shell - `install-capi.sh`: POSIX-compliant shell script to download and install PineAPPL's pre-built CAPI @@ -63,9 +59,6 @@ [lock file]: https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html [license v3]: https://www.gnu.org/licenses/gpl-3.0.en.html [PineAPPL's website]: https://nnpdf.github.io/pineappl/ -[flake]: https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html#flake-format -[flake lock file]: https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html#lock-files -[direnv]: https://direnv.net/ ## Abbreviations and terminology diff --git a/flake.lock b/flake.lock deleted file mode 100644 index 3732b67e5..000000000 --- a/flake.lock +++ /dev/null @@ -1,376 +0,0 @@ -{ - "nodes": { - "devenv": { - "inputs": { - "flake-compat": "flake-compat", - "nix": "nix", - "nixpkgs": "nixpkgs", - "pre-commit-hooks": "pre-commit-hooks" - }, - "locked": { - "lastModified": 1691413946, - "narHash": "sha256-IQ9gWBnUMJaXXev9vjnUJDfX0y4tOprGF773xD0rwHA=", - "owner": "cachix", - "repo": "devenv", - "rev": "b4ce82b6a54022c688f9f916b9f7a9c9ed959de1", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "devenv", - "type": "github" - } - }, - "fenix": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ], - "rust-analyzer-src": "rust-analyzer-src" - }, - "locked": { - "lastModified": 1691475627, - "narHash": "sha256-aOgYmNqXyCHw1vaNDv5GqUAURiPMVej4xQNE37bCF8g=", - "owner": "nix-community", - "repo": "fenix", - "rev": "72b64bd857cacd2733ce99fbf0603bbe335d357c", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "fenix", - "type": "github" - } - }, - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-compat_2": { - "flake": false, - "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1685518550, - "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_2": { - "inputs": { - "systems": "systems_2" - }, - "locked": { - "lastModified": 1685518550, - "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", - "type": "github" - }, - "original": { - "id": "flake-utils", - "type": "indirect" - } - }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "devenv", - "pre-commit-hooks", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1660459072, - "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, - "lowdown-src": { - "flake": false, - "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, - "nix": { - "inputs": { - "lowdown-src": "lowdown-src", - "nixpkgs": [ - "devenv", - "nixpkgs" - ], - "nixpkgs-regression": "nixpkgs-regression" - }, - "locked": { - "lastModified": 1676545802, - "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", - "owner": "domenkozar", - "repo": "nix", - "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", - "type": "github" - }, - "original": { - "owner": "domenkozar", - "ref": "relaxed-flakes", - "repo": "nix", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1678875422, - "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-python": { - "inputs": { - "flake-compat": "flake-compat_2", - "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs_3" - }, - "locked": { - "lastModified": 1690216544, - "narHash": "sha256-fWZVYvyoGTNzRnr6MDd0O2FLA2GwLCkSOlZbOxEBuk0=", - "owner": "cachix", - "repo": "nixpkgs-python", - "rev": "64b32e82cba00c8a1fdc53618ae8b5502866fe96", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "nixpkgs-python", - "type": "github" - } - }, - "nixpkgs-regression": { - "locked": { - "lastModified": 1643052045, - "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - } - }, - "nixpkgs-stable": { - "locked": { - "lastModified": 1685801374, - "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-23.05", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1696697597, - "narHash": "sha256-q26Qv4DQ+h6IeozF2o1secyQG0jt2VUT3V0K58jr3pg=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "5a237aecb57296f67276ac9ab296a41c23981f56", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-23.05", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_3": { - "locked": { - "lastModified": 1685974512, - "narHash": "sha256-WLPHpe96RbPRO9iDtCxgsYkadTheRq7wqXWdGpR6g7w=", - "owner": "domenkozar", - "repo": "nixpkgs", - "rev": "1102477695918daba466123cc2ef694ed3a49939", - "type": "github" - }, - "original": { - "owner": "domenkozar", - "ref": "cpython-moduralize", - "repo": "nixpkgs", - "type": "github" - } - }, - "pre-commit-hooks": { - "inputs": { - "flake-compat": [ - "devenv", - "flake-compat" - ], - "flake-utils": "flake-utils", - "gitignore": "gitignore", - "nixpkgs": [ - "devenv", - "nixpkgs" - ], - "nixpkgs-stable": "nixpkgs-stable" - }, - "locked": { - "lastModified": 1688056373, - "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "type": "github" - } - }, - "root": { - "inputs": { - "devenv": "devenv", - "fenix": "fenix", - "nixpkgs": "nixpkgs_2", - "nixpkgs-python": "nixpkgs-python", - "systems": "systems_3" - } - }, - "rust-analyzer-src": { - "flake": false, - "locked": { - "lastModified": 1691410619, - "narHash": "sha256-npJ8zOZ/CQoZu+0kN5NU9iC5JbmbsucyK1DayPmHWPs=", - "owner": "rust-lang", - "repo": "rust-analyzer", - "rev": "d9e9ca29814e95bb40db2216ed0973d9ba54b6a2", - "type": "github" - }, - "original": { - "owner": "rust-lang", - "ref": "nightly", - "repo": "rust-analyzer", - "type": "github" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "systems_2": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "systems_3": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index ea008d414..000000000 --- a/flake.nix +++ /dev/null @@ -1,70 +0,0 @@ -{ - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - systems.url = "github:nix-systems/default"; - devenv.url = "github:cachix/devenv"; - nixpkgs-python.url = "github:cachix/nixpkgs-python"; - fenix.url = "github:nix-community/fenix"; - fenix.inputs.nixpkgs.follows = "nixpkgs"; - }; - - outputs = { - self, - nixpkgs, - devenv, - systems, - ... - } @ inputs: let - forEachSystem = nixpkgs.lib.genAttrs (import systems); - in { - devShells = - forEachSystem - (system: let - pkgs = nixpkgs.legacyPackages.${system}; - pwd = builtins.getEnv "PWD"; - prefix = "${pwd}/target/prefix"; - lhapath = "${prefix}/share/LHAPDF"; - in { - default = devenv.lib.mkShell { - inherit inputs pkgs; - modules = [ - ({config, ...}: { - packages = with pkgs; [ - maturin - (lhapdf.override { - python = - config.languages.python.package; - }) - ]; - - env = { - PREFIX = prefix; - LHAPDF_DATA_PATH = lhapath; - }; - enterShell = '' - # update path before entering the shell, when Nix packages updates - # already happened - export PATH=${prefix}/bin:$PATH - mkdir -p ${lhapath} - ''; - - languages.python = { - enable = true; - venv.enable = true; - version = "3.11"; - }; - languages.rust = { - enable = true; - channel = "stable"; - }; - }) - ]; - }; - }); - }; - - nixConfig = { - extra-trusted-public-keys = "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw="; - extra-substituters = "https://devenv.cachix.org"; - }; -} diff --git a/maintainer/.gitignore b/maintainer/.gitignore deleted file mode 100644 index c18dd8d83..000000000 --- a/maintainer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__/ diff --git a/maintainer/install_lhapdf.py b/maintainer/install_lhapdf.py deleted file mode 100644 index a394b2cee..000000000 --- a/maintainer/install_lhapdf.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Install LHAPDF.""" - -import io -import logging -import os -import sys -import tarfile -import tempfile -from http.client import HTTPSConnection -from pathlib import Path -from subprocess import run - -VERSION = "6.5.3" -TARBALL = f"LHAPDF-{VERSION}.tar.gz" -HOST = "lhapdf.hepforge.org" -PATH = f"/downloads/?f={TARBALL}" -SRC_DIR = f"LHAPDF-{VERSION}" - - -logging.basicConfig() -log = logging.getLogger("LHAPDF installer") -log.setLevel(logging.INFO) - - -def download(host: str, path: str) -> bytes: - """Download content from remote server.""" - conn = HTTPSConnection(host) - conn.request("GET", path) - resp = conn.getresponse() - return resp.read() - - -def install(path: Path): - """Build from source and install.""" - - def run_(cmd, *args, **kwargs): - cmd = cmd.split() if isinstance(cmd, str) else cmd - run(cmd, *args, cwd=path / SRC_DIR, **kwargs) - - # log.info("Autoreconf...") - # run_("autoreconf -f -i") - prefix = os.environ.get("PREFIX") - prefix_ = ["--prefix", prefix] if prefix is not None else [] - log.info("Configuring...") - run_(["./configure"] + prefix_, env={"PYTHON": sys.executable}) - log.info("Building...") - run_("make clean") - run_("make -j") - log.info("Installing...") - run_("make install") - log.info("Installed") - - -def main(): - """Install LHAPDF.""" - tar = download(HOST, PATH) - log.info(f"Downloaded {HOST}{PATH}") - with tempfile.TemporaryDirectory() as tmpd: - tarfile.open(fileobj=io.BytesIO(tar), mode="r:gz").extractall(tmpd) - log.info(f"Extracted LHAPDF tarball in {tmpd}") - install(Path(tmpd)) - - -if __name__ == "__main__": - main() diff --git a/maintainer/patch_binary_wheel.py b/maintainer/patch_binary_wheel.py deleted file mode 100644 index 00da4be6b..000000000 --- a/maintainer/patch_binary_wheel.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Patch binary wheel to allow executable binaries. - -Following the patch in https://github.com/pypa/auditwheel/pull/443/ -""" - -import base64 -import hashlib -import logging -import shutil -import sys -import tempfile -from pathlib import Path - -logging.basicConfig() -log = logging.getLogger("binary-patch") -log.setLevel(logging.INFO) - -WRAPPER = """\ -#!python -import os -import sys -import sysconfig - -if __name__ == "__main__": - os.execv( - os.path.join(sysconfig.get_path("platlib"), {binary_path!r}), - sys.argv, - ) -""" - - -def hash(path: Path): - """Compute file hash.""" - return ( - base64.urlsafe_b64encode(hashlib.sha256(path.read_bytes()).digest()) - .decode() - .rstrip("=") - ) - - -def record(tmpd): - """Generate a RECORD file. - - Rehash the content of the wheel. - """ - lines = [] - for path in Path(tmpd).glob("**/*"): - if not path.is_file(): - continue - - path_ = path.relative_to(tmpd) - if path.name == "RECORD": - lines.append(f"{path_},,") - continue - - hash_ = hash(path) - size = path.stat().st_size - lines.append(f"{path_},sha256={hash_},{size}") - - return "\n".join(lines) - - -def patch(wheel: Path): - """Patch wheel to add wrapper script.""" - with tempfile.TemporaryDirectory() as tmpd: - shutil.unpack_archive(wheel, tmpd, format="zip") - scripts = Path(tmpd) / "pineappl_cli.scripts" - scripts.mkdir() - installed = next(Path(tmpd).glob("*.data")) / "scripts" / "pineappl" - shutil.move(installed, scripts) - binary = scripts / "pineappl" - installed.write_text(WRAPPER.format(binary_path=str(binary.relative_to(tmpd)))) - next(Path(tmpd).glob("**/RECORD")).write_text(record(tmpd)) - shutil.make_archive(wheel.name, format="zip", base_dir=".", root_dir=tmpd) - shutil.move(wheel.name + ".zip", wheel) - log.info(f"Patched {wheel} in-place") - - -def main(): - """Patch wheels passed as arguments.""" - wheels = [Path(p) for p in sys.argv[1:]] - log.info("Processing the following wheels:\n" + "\n".join(str(w) for w in wheels)) - for wheel in wheels: - patch(wheel) - - -if __name__ == "__main__": - main() From 295a2755f2ff8c948d4b48b1fb5be15b3f766fd6 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 09:37:31 +0200 Subject: [PATCH 83/92] Rename CLI wheel for Windows workflow file --- .../{cli-wheels.yml => cli-wheels-windows.yml} | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) rename .github/workflows/{cli-wheels.yml => cli-wheels-windows.yml} (93%) diff --git a/.github/workflows/cli-wheels.yml b/.github/workflows/cli-wheels-windows.yml similarity index 93% rename from .github/workflows/cli-wheels.yml rename to .github/workflows/cli-wheels-windows.yml index 3f6dcf361..52655c73f 100644 --- a/.github/workflows/cli-wheels.yml +++ b/.github/workflows/cli-wheels-windows.yml @@ -1,21 +1,16 @@ -name: Publish CLI wheels +name: Generate CLI wheels for Windows + +# this workflow doesn't do (yet) what it promises; we keep this file to not lose the progress on: - push: - # tags: - # - "v[0-9]+*" workflow_dispatch: -env: - GH_TOKEN: ${{ github.token }} - jobs: - wheels-windows: + cli-wheels-windows: runs-on: windows-latest strategy: matrix: target: [x64] - # if: false defaults: defaults: run: shell: msys2 {0} From e408b2b61038de69c11c733692c74495bceb5bf3 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 10:01:03 +0200 Subject: [PATCH 84/92] Publish also CLI wheels --- .github/workflows/release.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8b77142bb..08741ac76 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -393,11 +393,17 @@ jobs: release-wheels: runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" - needs: [wheels-linux, wheels-macos, wheels-sdist, wheels-windows] + needs: + - cli-wheels-linux + - cli-wheels-macos + - wheels-linux + - wheels-macos + - wheels-sdist + - wheels-windows steps: - uses: actions/download-artifact@v4 with: - pattern: wheels-* + pattern: "*wheels-*" merge-multiple: true - name: Publish to PyPI uses: PyO3/maturin-action@v1 From c0323257c5ea8937284934fe56eae44237b3f019 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 11:02:24 +0200 Subject: [PATCH 85/92] Make wheels-only release --- .github/workflows/release.yml | 362 +++++++++++++++++----------------- 1 file changed, 181 insertions(+), 181 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 08741ac76..4dea4b588 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,88 +11,88 @@ env: GH_TOKEN: ${{ github.token }} jobs: - # create a release on github - create-release: - runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/')" - steps: - - uses: actions/checkout@v4 - - name: Create new release - run: | - # remove leading 'v' in tag name - gh release create -d v${GITHUB_REF_NAME#v} + ## create a release on github + #create-release: + # runs-on: ubuntu-latest + # if: "startsWith(github.ref, 'refs/tags/')" + # steps: + # - uses: actions/checkout@v4 + # - name: Create new release + # run: | + # # remove leading 'v' in tag name + # gh release create -d v${GITHUB_REF_NAME#v} - capi-macos: - runs-on: macos-latest - strategy: - matrix: - target: [x86_64-apple-darwin, aarch64-apple-darwin] - steps: - - uses: actions/checkout@v4 - - name: Compile library - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --target=${{ matrix.target }} - cargo install --locked cargo-c - cd pineappl_capi - cargo cinstall --destdir=prefix --library-type=cdylib --locked --prefix=/ --target=${{ matrix.target }} --verbose - cd prefix - tar czf ../../pineappl_capi-${{ matrix.target }}.tar.gz . - - name: Upload artifact - # as long as we need v3 in `capi-linux` we also must use it here - uses: actions/upload-artifact@v3 - with: - name: pineappl_capi-${{ matrix.target }} - path: pineappl_capi-${{ matrix.target }}.tar.gz + #capi-macos: + # runs-on: macos-latest + # strategy: + # matrix: + # target: [x86_64-apple-darwin, aarch64-apple-darwin] + # steps: + # - uses: actions/checkout@v4 + # - name: Compile library + # run: | + # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --target=${{ matrix.target }} + # cargo install --locked cargo-c + # cd pineappl_capi + # cargo cinstall --destdir=prefix --library-type=cdylib --locked --prefix=/ --target=${{ matrix.target }} --verbose + # cd prefix + # tar czf ../../pineappl_capi-${{ matrix.target }}.tar.gz . + # - name: Upload artifact + # # as long as we need v3 in `capi-linux` we also must use it here + # uses: actions/upload-artifact@v3 + # with: + # name: pineappl_capi-${{ matrix.target }} + # path: pineappl_capi-${{ matrix.target }}.tar.gz - cli-macos: - strategy: - matrix: - include: - - os: macos-13 - target: x86_64-apple-darwin - - os: macos-14 - # we cannot cross-compile, because we need also all dependencies for the specified target - target: aarch64-apple-darwin - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Compile binary - run: | - brew tap davidchall/hep - # install LHAPDF - brew install lhapdf - # install APPLgrid's dependencies; disable ROOT because its static libraries are missing - brew install --only-dependencies --without-hoppet --without-lhapdf --without-root applgrid - # install zlib, which is a dependency of APPLgrid but somehow missing - brew install zlib - # APPLgrid wants to be linked against zlib, and we need to find its static library via pkg-config - export PKG_CONFIG_PATH=$(find $(brew --cellar) -name '*.pc' -exec dirname {} + | sort -u | tr '\n' ':') - # manually compile APPLgrid, because we need the file `appl_igrid.h` and the files it includes, which are possibly generated - export HOMEBREW_TEMP="$(pwd)"/tmp - mkdir -p ${HOMEBREW_TEMP} - brew install --build-from-source --keep-tmp --without-hoppet --without-lhapdf --without-root applgrid - export APPL_IGRID_DIR=$(find ${HOMEBREW_TEMP} -name appl_igrid.h -exec dirname {} +) - # do not link against `gfortran` - sed 's/-lgfortran//g' $(command -v applgrid-config) > applgrid-config.new - mv applgrid-config.new $(command -v applgrid-config) - chmod +x $(command -v applgrid-config) - # install fastNLO - brew install --without-fastjet fastnlo - # install Rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-host=${{ matrix.target }} - # build the CLI - cargo install --all-features --locked --path pineappl_cli --root=prefix --target=${{ matrix.target }} - # build manpages - mkdir -p prefix/share/man/man1 - cargo xtask install-manpages prefix/share/man/man1 - cd prefix - tar czf ../pineappl_cli-${{ matrix.target }}.tar.gz . - - name: Upload artifact - # as long as we need v3 in `cli-linux` we also must use it here - uses: actions/upload-artifact@v3 - with: - name: pineappl_cli-${{ matrix.target }} - path: pineappl_cli-${{ matrix.target }}.tar.gz + #cli-macos: + # strategy: + # matrix: + # include: + # - os: macos-13 + # target: x86_64-apple-darwin + # - os: macos-14 + # # we cannot cross-compile, because we need also all dependencies for the specified target + # target: aarch64-apple-darwin + # runs-on: ${{ matrix.os }} + # steps: + # - uses: actions/checkout@v4 + # - name: Compile binary + # run: | + # brew tap davidchall/hep + # # install LHAPDF + # brew install lhapdf + # # install APPLgrid's dependencies; disable ROOT because its static libraries are missing + # brew install --only-dependencies --without-hoppet --without-lhapdf --without-root applgrid + # # install zlib, which is a dependency of APPLgrid but somehow missing + # brew install zlib + # # APPLgrid wants to be linked against zlib, and we need to find its static library via pkg-config + # export PKG_CONFIG_PATH=$(find $(brew --cellar) -name '*.pc' -exec dirname {} + | sort -u | tr '\n' ':') + # # manually compile APPLgrid, because we need the file `appl_igrid.h` and the files it includes, which are possibly generated + # export HOMEBREW_TEMP="$(pwd)"/tmp + # mkdir -p ${HOMEBREW_TEMP} + # brew install --build-from-source --keep-tmp --without-hoppet --without-lhapdf --without-root applgrid + # export APPL_IGRID_DIR=$(find ${HOMEBREW_TEMP} -name appl_igrid.h -exec dirname {} +) + # # do not link against `gfortran` + # sed 's/-lgfortran//g' $(command -v applgrid-config) > applgrid-config.new + # mv applgrid-config.new $(command -v applgrid-config) + # chmod +x $(command -v applgrid-config) + # # install fastNLO + # brew install --without-fastjet fastnlo + # # install Rust + # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-host=${{ matrix.target }} + # # build the CLI + # cargo install --all-features --locked --path pineappl_cli --root=prefix --target=${{ matrix.target }} + # # build manpages + # mkdir -p prefix/share/man/man1 + # cargo xtask install-manpages prefix/share/man/man1 + # cd prefix + # tar czf ../pineappl_cli-${{ matrix.target }}.tar.gz . + # - name: Upload artifact + # # as long as we need v3 in `cli-linux` we also must use it here + # uses: actions/upload-artifact@v3 + # with: + # name: pineappl_cli-${{ matrix.target }} + # path: pineappl_cli-${{ matrix.target }}.tar.gz cli-wheels-macos: strategy: @@ -154,55 +154,55 @@ jobs: name: ${{ github.job }}-${{ matrix.target }} path: dist - capi-linux: - runs-on: ubuntu-latest - container: ghcr.io/nnpdf/pineappl-ci:latest - strategy: - matrix: - target: [x86_64-unknown-linux-gnu] - steps: - # checkout@v4 uses a newer version of Node that's incompatible with our container's GLIBC - - uses: actions/checkout@v3 - - name: Compile library - run: | - cd pineappl_capi - cargo cinstall --destdir=prefix --library-type=cdylib --locked --prefix=/ --target=${{ matrix.target }} --verbose - cd prefix - tar czf ../../pineappl_capi-${{ matrix.target }}.tar.gz . - # print the glibc version requirement - objdump -T lib/libpineappl_capi.so | grep GLIBC | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -1 - - name: Upload artifact - # upload-artifact@v4 uses a newer version of Node that's incompatible with our container's GLIBC - uses: actions/upload-artifact@v3 - with: - name: pineappl_capi-${{ matrix.target }} - path: pineappl_capi-${{ matrix.target }}.tar.gz + #capi-linux: + # runs-on: ubuntu-latest + # container: ghcr.io/nnpdf/pineappl-ci:latest + # strategy: + # matrix: + # target: [x86_64-unknown-linux-gnu] + # steps: + # # checkout@v4 uses a newer version of Node that's incompatible with our container's GLIBC + # - uses: actions/checkout@v3 + # - name: Compile library + # run: | + # cd pineappl_capi + # cargo cinstall --destdir=prefix --library-type=cdylib --locked --prefix=/ --target=${{ matrix.target }} --verbose + # cd prefix + # tar czf ../../pineappl_capi-${{ matrix.target }}.tar.gz . + # # print the glibc version requirement + # objdump -T lib/libpineappl_capi.so | grep GLIBC | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -1 + # - name: Upload artifact + # # upload-artifact@v4 uses a newer version of Node that's incompatible with our container's GLIBC + # uses: actions/upload-artifact@v3 + # with: + # name: pineappl_capi-${{ matrix.target }} + # path: pineappl_capi-${{ matrix.target }}.tar.gz - cli-linux: - runs-on: ubuntu-latest - container: ghcr.io/nnpdf/pineappl-ci:latest - strategy: - matrix: - target: [x86_64-unknown-linux-gnu] - steps: - # checkout@v4 uses a newer version of Node that's incompatible with our container's GLIBC - - uses: actions/checkout@v3 - - name: Compile binary - run: | - cargo install --all-features --locked --path pineappl_cli --root=prefix --target=${{ matrix.target }} -vv - # build manpages - mkdir -p prefix/share/man/man1 - cargo xtask install-manpages prefix/share/man/man1 - cd prefix - tar czf ../pineappl_cli-${{ matrix.target }}.tar.gz . - # print the dynamically linked libraries - ldd bin/pineappl - - name: Upload artifact - # upload-artifact@v4 uses a newer version of Node that's incompatible with our container's GLIBC - uses: actions/upload-artifact@v3 - with: - name: pineappl_cli-${{ matrix.target }} - path: pineappl_cli-${{ matrix.target }}.tar.gz + #cli-linux: + # runs-on: ubuntu-latest + # container: ghcr.io/nnpdf/pineappl-ci:latest + # strategy: + # matrix: + # target: [x86_64-unknown-linux-gnu] + # steps: + # # checkout@v4 uses a newer version of Node that's incompatible with our container's GLIBC + # - uses: actions/checkout@v3 + # - name: Compile binary + # run: | + # cargo install --all-features --locked --path pineappl_cli --root=prefix --target=${{ matrix.target }} -vv + # # build manpages + # mkdir -p prefix/share/man/man1 + # cargo xtask install-manpages prefix/share/man/man1 + # cd prefix + # tar czf ../pineappl_cli-${{ matrix.target }}.tar.gz . + # # print the dynamically linked libraries + # ldd bin/pineappl + # - name: Upload artifact + # # upload-artifact@v4 uses a newer version of Node that's incompatible with our container's GLIBC + # uses: actions/upload-artifact@v3 + # with: + # name: pineappl_cli-${{ matrix.target }} + # path: pineappl_cli-${{ matrix.target }}.tar.gz cli-wheels-linux: runs-on: ubuntu-latest @@ -226,61 +226,61 @@ jobs: name: ${{ github.job }}-${{ matrix.target }} path: dist - # publish a release on github and upload pre-built CAPI - publish-release: - needs: [capi-macos, cli-macos, capi-linux, cli-linux] - runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/')" - steps: - - uses: actions/checkout@v4 - # version must match the one used in `actions/upload-artifact` - - uses: actions/download-artifact@v3 - with: - path: artifacts - - name: Create new release - run: | - # remove leading 'v' in tag name - version=${GITHUB_REF_NAME#v} - prerelease=$(echo ${version} | perl -pe 's/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/\4/') - if [[ ${prerelease} == "" ]]; then - # extract the previous version number - old_version=$(sed -n 's/^## \[\(.*\)\] - .*/\1/p' CHANGELOG.md | tail +2 | head -n 1) - # extract news for the current version from the changelog file, dismissing - # empty lines at the start and the end - news=$(sed -n "/\\[${version}\\]/, /\\[${old_version}\\]/{ /\\[${old_version}\\]/! p }" \ - CHANGELOG.md | sed -e :a -e '/./,$!d;/^\n*$/{$d;N;};/\n$/ba') - gh release edit v${version} -n "${news}" - else - gh release edit v${version} -n "" --prerelease - fi - find artifacts -name 'pineappl*' -type f -exec gh release upload v${version} {} + - gh release edit v${version} --draft=false + ## publish a release on github and upload pre-built CAPI + #publish-release: + # needs: [capi-macos, cli-macos, capi-linux, cli-linux] + # runs-on: ubuntu-latest + # if: "startsWith(github.ref, 'refs/tags/')" + # steps: + # - uses: actions/checkout@v4 + # # version must match the one used in `actions/upload-artifact` + # - uses: actions/download-artifact@v3 + # with: + # path: artifacts + # - name: Create new release + # run: | + # # remove leading 'v' in tag name + # version=${GITHUB_REF_NAME#v} + # prerelease=$(echo ${version} | perl -pe 's/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/\4/') + # if [[ ${prerelease} == "" ]]; then + # # extract the previous version number + # old_version=$(sed -n 's/^## \[\(.*\)\] - .*/\1/p' CHANGELOG.md | tail +2 | head -n 1) + # # extract news for the current version from the changelog file, dismissing + # # empty lines at the start and the end + # news=$(sed -n "/\\[${version}\\]/, /\\[${old_version}\\]/{ /\\[${old_version}\\]/! p }" \ + # CHANGELOG.md | sed -e :a -e '/./,$!d;/^\n*$/{$d;N;};/\n$/ba') + # gh release edit v${version} -n "${news}" + # else + # gh release edit v${version} -n "" --prerelease + # fi + # find artifacts -name 'pineappl*' -type f -exec gh release upload v${version} {} + + # gh release edit v${version} --draft=false - publish-crates: - runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/')" - container: ghcr.io/nnpdf/pineappl-ci:latest - steps: - # checkout@v4 uses a newer version of Node that's incompatible with our container's GLIBC - - uses: actions/checkout@v3 + #publish-crates: + # runs-on: ubuntu-latest + # if: "startsWith(github.ref, 'refs/tags/')" + # container: ghcr.io/nnpdf/pineappl-ci:latest + # steps: + # # checkout@v4 uses a newer version of Node that's incompatible with our container's GLIBC + # - uses: actions/checkout@v3 - - name: Publish all crates - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - run: | - # this must be at least Rust 1.66 to support waiting for dependencies to on crates.io: - # https://github.com/rust-lang/cargo/blob/master/CHANGELOG.md#cargo-166-2022-12-15 - rustup default stable - cd pineappl - cargo publish - cd ../pineappl_applgrid - cargo publish - cd ../pineappl_fastnlo - cargo publish - cd ../pineappl_capi - cargo publish - cd ../pineappl_cli - cargo publish + # - name: Publish all crates + # env: + # CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + # run: | + # # this must be at least Rust 1.66 to support waiting for dependencies to on crates.io: + # # https://github.com/rust-lang/cargo/blob/master/CHANGELOG.md#cargo-166-2022-12-15 + # rustup default stable + # cd pineappl + # cargo publish + # cd ../pineappl_applgrid + # cargo publish + # cd ../pineappl_fastnlo + # cargo publish + # cd ../pineappl_capi + # cargo publish + # cd ../pineappl_cli + # cargo publish wheels-linux: runs-on: ubuntu-latest From 0e45d2f1c618d216f53eb2dda65a313dcebba9aa Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 11:11:31 +0200 Subject: [PATCH 86/92] Release v0.8.0-alpha.1.testwheels.1 --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- pineappl_capi/Cargo.toml | 2 +- pineappl_cli/Cargo.toml | 6 +++--- pineappl_py/Cargo.toml | 2 +- xtask/Cargo.toml | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d70e32868..88eff845c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,7 +1225,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pineappl" -version = "0.7.4" +version = "0.8.0-alpha.1.testwheels.1" dependencies = [ "anyhow", "arrayvec", @@ -1250,7 +1250,7 @@ dependencies = [ [[package]] name = "pineappl_applgrid" -version = "0.7.4" +version = "0.8.0-alpha.1.testwheels.1" dependencies = [ "cc", "cxx", @@ -1261,7 +1261,7 @@ dependencies = [ [[package]] name = "pineappl_capi" -version = "0.7.4" +version = "0.8.0-alpha.1.testwheels.1" dependencies = [ "itertools", "pineappl", @@ -1269,7 +1269,7 @@ dependencies = [ [[package]] name = "pineappl_cli" -version = "0.7.4" +version = "0.8.0-alpha.1.testwheels.1" dependencies = [ "anyhow", "assert_cmd", @@ -1300,7 +1300,7 @@ dependencies = [ [[package]] name = "pineappl_fastnlo" -version = "0.7.4" +version = "0.8.0-alpha.1.testwheels.1" dependencies = [ "cxx", "cxx-build", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "pineappl_py" -version = "0.7.4" +version = "0.8.0-alpha.1.testwheels.1" dependencies = [ "itertools", "ndarray", @@ -2472,7 +2472,7 @@ dependencies = [ [[package]] name = "xtask" -version = "0.7.4" +version = "0.8.0-alpha.1.testwheels.1" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index 6aa82170d..aa3e8e985 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ keywords = ["high-energy-physics", "physics"] license = "GPL-3.0-or-later" repository = "https://github.com/NNPDF/pineappl" rust-version = "1.70.0" -version = "0.7.4" +version = "0.8.0-alpha.1.testwheels.1" [workspace.lints.clippy] all = { level = "warn", priority = -1 } diff --git a/pineappl_capi/Cargo.toml b/pineappl_capi/Cargo.toml index 864faf214..af8c01e98 100644 --- a/pineappl_capi/Cargo.toml +++ b/pineappl_capi/Cargo.toml @@ -16,7 +16,7 @@ version.workspace = true workspace = true [dependencies] -pineappl = { path = "../pineappl", version = "=0.7.4" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.1.testwheels.1" } itertools = "0.10.1" [features] diff --git a/pineappl_cli/Cargo.toml b/pineappl_cli/Cargo.toml index 461729365..51078fb38 100644 --- a/pineappl_cli/Cargo.toml +++ b/pineappl_cli/Cargo.toml @@ -30,9 +30,9 @@ lhapdf = { package = "managed-lhapdf", version = "0.3.2" } lz4_flex = { optional = true, version = "0.9.2" } ndarray = "0.15.4" ndarray-npy = { optional = true, version = "0.8.1" } -pineappl = { path = "../pineappl", version = "=0.7.4" } -pineappl_applgrid = { optional = true, path = "../pineappl_applgrid", version = "=0.7.4" } -pineappl_fastnlo = { optional = true, path = "../pineappl_fastnlo", version = "=0.7.4" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.1.testwheels.1" } +pineappl_applgrid = { optional = true, path = "../pineappl_applgrid", version = "=0.8.0-alpha.1.testwheels.1" } +pineappl_fastnlo = { optional = true, path = "../pineappl_fastnlo", version = "=0.8.0-alpha.1.testwheels.1" } prettytable-rs = { default-features = false, features = ["win_crlf"], version = "0.10.0" } rayon = "1.5.1" serde = { features = ["derive"], optional = true, version = "1.0.130" } diff --git a/pineappl_py/Cargo.toml b/pineappl_py/Cargo.toml index 5b7f35b63..e50e34236 100644 --- a/pineappl_py/Cargo.toml +++ b/pineappl_py/Cargo.toml @@ -30,5 +30,5 @@ crate-type = ["cdylib"] itertools = "0.10.1" ndarray = "0.15.4" numpy = "0.20.0" -pineappl = { path = "../pineappl", version = "=0.7.4" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.1.testwheels.1" } pyo3 = { features = ["extension-module"], version = "0.20.0" } diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 8b025b56e..befea6179 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -21,4 +21,4 @@ clap_mangen = "0.2.18" enum_dispatch = "0.3.7" #git2 = "0.17.2" #semver = "1.0.17" -pineappl_cli = { path = "../pineappl_cli", version = "=0.7.4" } +pineappl_cli = { path = "../pineappl_cli", version = "=0.8.0-alpha.1.testwheels.1" } From 7400237de1160352bf686aedb97f6b9cf087b072 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 11:13:21 +0200 Subject: [PATCH 87/92] Release v0.8.0-alpha.2 --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- pineappl_capi/Cargo.toml | 2 +- pineappl_cli/Cargo.toml | 6 +++--- pineappl_py/Cargo.toml | 2 +- xtask/Cargo.toml | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88eff845c..d4197828b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,7 +1225,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pineappl" -version = "0.8.0-alpha.1.testwheels.1" +version = "0.8.0-alpha.2" dependencies = [ "anyhow", "arrayvec", @@ -1250,7 +1250,7 @@ dependencies = [ [[package]] name = "pineappl_applgrid" -version = "0.8.0-alpha.1.testwheels.1" +version = "0.8.0-alpha.2" dependencies = [ "cc", "cxx", @@ -1261,7 +1261,7 @@ dependencies = [ [[package]] name = "pineappl_capi" -version = "0.8.0-alpha.1.testwheels.1" +version = "0.8.0-alpha.2" dependencies = [ "itertools", "pineappl", @@ -1269,7 +1269,7 @@ dependencies = [ [[package]] name = "pineappl_cli" -version = "0.8.0-alpha.1.testwheels.1" +version = "0.8.0-alpha.2" dependencies = [ "anyhow", "assert_cmd", @@ -1300,7 +1300,7 @@ dependencies = [ [[package]] name = "pineappl_fastnlo" -version = "0.8.0-alpha.1.testwheels.1" +version = "0.8.0-alpha.2" dependencies = [ "cxx", "cxx-build", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "pineappl_py" -version = "0.8.0-alpha.1.testwheels.1" +version = "0.8.0-alpha.2" dependencies = [ "itertools", "ndarray", @@ -2472,7 +2472,7 @@ dependencies = [ [[package]] name = "xtask" -version = "0.8.0-alpha.1.testwheels.1" +version = "0.8.0-alpha.2" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index aa3e8e985..aba3624a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ keywords = ["high-energy-physics", "physics"] license = "GPL-3.0-or-later" repository = "https://github.com/NNPDF/pineappl" rust-version = "1.70.0" -version = "0.8.0-alpha.1.testwheels.1" +version = "0.8.0-alpha.2" [workspace.lints.clippy] all = { level = "warn", priority = -1 } diff --git a/pineappl_capi/Cargo.toml b/pineappl_capi/Cargo.toml index af8c01e98..bed3b610e 100644 --- a/pineappl_capi/Cargo.toml +++ b/pineappl_capi/Cargo.toml @@ -16,7 +16,7 @@ version.workspace = true workspace = true [dependencies] -pineappl = { path = "../pineappl", version = "=0.8.0-alpha.1.testwheels.1" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.2" } itertools = "0.10.1" [features] diff --git a/pineappl_cli/Cargo.toml b/pineappl_cli/Cargo.toml index 51078fb38..94e3749b7 100644 --- a/pineappl_cli/Cargo.toml +++ b/pineappl_cli/Cargo.toml @@ -30,9 +30,9 @@ lhapdf = { package = "managed-lhapdf", version = "0.3.2" } lz4_flex = { optional = true, version = "0.9.2" } ndarray = "0.15.4" ndarray-npy = { optional = true, version = "0.8.1" } -pineappl = { path = "../pineappl", version = "=0.8.0-alpha.1.testwheels.1" } -pineappl_applgrid = { optional = true, path = "../pineappl_applgrid", version = "=0.8.0-alpha.1.testwheels.1" } -pineappl_fastnlo = { optional = true, path = "../pineappl_fastnlo", version = "=0.8.0-alpha.1.testwheels.1" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.2" } +pineappl_applgrid = { optional = true, path = "../pineappl_applgrid", version = "=0.8.0-alpha.2" } +pineappl_fastnlo = { optional = true, path = "../pineappl_fastnlo", version = "=0.8.0-alpha.2" } prettytable-rs = { default-features = false, features = ["win_crlf"], version = "0.10.0" } rayon = "1.5.1" serde = { features = ["derive"], optional = true, version = "1.0.130" } diff --git a/pineappl_py/Cargo.toml b/pineappl_py/Cargo.toml index e50e34236..6ee627705 100644 --- a/pineappl_py/Cargo.toml +++ b/pineappl_py/Cargo.toml @@ -30,5 +30,5 @@ crate-type = ["cdylib"] itertools = "0.10.1" ndarray = "0.15.4" numpy = "0.20.0" -pineappl = { path = "../pineappl", version = "=0.8.0-alpha.1.testwheels.1" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.2" } pyo3 = { features = ["extension-module"], version = "0.20.0" } diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index befea6179..627d193e9 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -21,4 +21,4 @@ clap_mangen = "0.2.18" enum_dispatch = "0.3.7" #git2 = "0.17.2" #semver = "1.0.17" -pineappl_cli = { path = "../pineappl_cli", version = "=0.8.0-alpha.1.testwheels.1" } +pineappl_cli = { path = "../pineappl_cli", version = "=0.8.0-alpha.2" } From 6e71ec746decaf028d99850633b427830176a83a Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 11:27:43 +0200 Subject: [PATCH 88/92] Allow prereleases from every branch --- maintainer/make-release.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/maintainer/make-release.sh b/maintainer/make-release.sh index 12b7feb66..74ce49421 100755 --- a/maintainer/make-release.sh +++ b/maintainer/make-release.sh @@ -19,6 +19,9 @@ features=( fktable ) +main=master +this_branch=$(git rev-parse --abbrev-ref HEAD) + cd .. if [[ $# != 1 ]]; then @@ -28,13 +31,17 @@ fi version=$1 +prerelease=$(echo ${version} | perl -pe 's/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/\4/') + if [[ $(echo ${version} | grep -oP '^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$') != ${version} ]]; then echo "Version string incorrect." exit 1 fi -if [[ $(git rev-parse --abbrev-ref HEAD) != master ]]; then - echo "You're not on master." +# in branches that are not master we only allow prereleases +if [[ ${this_branch} != ${main} ]] && [[ ${prerelease} == "" ]]; then + echo "Ordinary releases are only allowed in the '${main}' branch." + echo "If you really want to make a release from '${this_branch}', consider making a prerelease." exit 1 fi @@ -73,8 +80,6 @@ cd .. echo ">>> Updating version strings ..." -prerelease=$(echo ${version} | perl -pe 's/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/\4/') - # we don't want to create a changelog entry for prereleases, which are solely # for internal testing purposes if [[ ${prerelease} == "" ]]; then From 7e5213b6f462214f0ee045e036fe7024c4fb723c Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 11:34:56 +0200 Subject: [PATCH 89/92] Try to fix cli-wheels not being downloaded in CI --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4dea4b588..ee9ae73cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -149,7 +149,7 @@ jobs: args: --release --out dist --find-interpreter --manifest-path pineappl_cli/Cargo.toml sccache: "true" - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-${{ matrix.target }} path: dist @@ -221,7 +221,7 @@ jobs: sccache: "true" manylinux: auto - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-${{ matrix.target }} path: dist From 8ff4a705551543a6d8085765a45682dc376c6031 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 11:36:46 +0200 Subject: [PATCH 90/92] Release v0.8.0-alpha.3 --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- pineappl_capi/Cargo.toml | 2 +- pineappl_cli/Cargo.toml | 6 +++--- pineappl_py/Cargo.toml | 2 +- xtask/Cargo.toml | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4197828b..6c7881009 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,7 +1225,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pineappl" -version = "0.8.0-alpha.2" +version = "0.8.0-alpha.3" dependencies = [ "anyhow", "arrayvec", @@ -1250,7 +1250,7 @@ dependencies = [ [[package]] name = "pineappl_applgrid" -version = "0.8.0-alpha.2" +version = "0.8.0-alpha.3" dependencies = [ "cc", "cxx", @@ -1261,7 +1261,7 @@ dependencies = [ [[package]] name = "pineappl_capi" -version = "0.8.0-alpha.2" +version = "0.8.0-alpha.3" dependencies = [ "itertools", "pineappl", @@ -1269,7 +1269,7 @@ dependencies = [ [[package]] name = "pineappl_cli" -version = "0.8.0-alpha.2" +version = "0.8.0-alpha.3" dependencies = [ "anyhow", "assert_cmd", @@ -1300,7 +1300,7 @@ dependencies = [ [[package]] name = "pineappl_fastnlo" -version = "0.8.0-alpha.2" +version = "0.8.0-alpha.3" dependencies = [ "cxx", "cxx-build", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "pineappl_py" -version = "0.8.0-alpha.2" +version = "0.8.0-alpha.3" dependencies = [ "itertools", "ndarray", @@ -2472,7 +2472,7 @@ dependencies = [ [[package]] name = "xtask" -version = "0.8.0-alpha.2" +version = "0.8.0-alpha.3" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index aba3624a0..99429e62f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ keywords = ["high-energy-physics", "physics"] license = "GPL-3.0-or-later" repository = "https://github.com/NNPDF/pineappl" rust-version = "1.70.0" -version = "0.8.0-alpha.2" +version = "0.8.0-alpha.3" [workspace.lints.clippy] all = { level = "warn", priority = -1 } diff --git a/pineappl_capi/Cargo.toml b/pineappl_capi/Cargo.toml index bed3b610e..ae564221f 100644 --- a/pineappl_capi/Cargo.toml +++ b/pineappl_capi/Cargo.toml @@ -16,7 +16,7 @@ version.workspace = true workspace = true [dependencies] -pineappl = { path = "../pineappl", version = "=0.8.0-alpha.2" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.3" } itertools = "0.10.1" [features] diff --git a/pineappl_cli/Cargo.toml b/pineappl_cli/Cargo.toml index 94e3749b7..94b64c64c 100644 --- a/pineappl_cli/Cargo.toml +++ b/pineappl_cli/Cargo.toml @@ -30,9 +30,9 @@ lhapdf = { package = "managed-lhapdf", version = "0.3.2" } lz4_flex = { optional = true, version = "0.9.2" } ndarray = "0.15.4" ndarray-npy = { optional = true, version = "0.8.1" } -pineappl = { path = "../pineappl", version = "=0.8.0-alpha.2" } -pineappl_applgrid = { optional = true, path = "../pineappl_applgrid", version = "=0.8.0-alpha.2" } -pineappl_fastnlo = { optional = true, path = "../pineappl_fastnlo", version = "=0.8.0-alpha.2" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.3" } +pineappl_applgrid = { optional = true, path = "../pineappl_applgrid", version = "=0.8.0-alpha.3" } +pineappl_fastnlo = { optional = true, path = "../pineappl_fastnlo", version = "=0.8.0-alpha.3" } prettytable-rs = { default-features = false, features = ["win_crlf"], version = "0.10.0" } rayon = "1.5.1" serde = { features = ["derive"], optional = true, version = "1.0.130" } diff --git a/pineappl_py/Cargo.toml b/pineappl_py/Cargo.toml index 6ee627705..75a682421 100644 --- a/pineappl_py/Cargo.toml +++ b/pineappl_py/Cargo.toml @@ -30,5 +30,5 @@ crate-type = ["cdylib"] itertools = "0.10.1" ndarray = "0.15.4" numpy = "0.20.0" -pineappl = { path = "../pineappl", version = "=0.8.0-alpha.2" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.3" } pyo3 = { features = ["extension-module"], version = "0.20.0" } diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 627d193e9..1b9e53cbb 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -21,4 +21,4 @@ clap_mangen = "0.2.18" enum_dispatch = "0.3.7" #git2 = "0.17.2" #semver = "1.0.17" -pineappl_cli = { path = "../pineappl_cli", version = "=0.8.0-alpha.2" } +pineappl_cli = { path = "../pineappl_cli", version = "=0.8.0-alpha.3" } From cdeadc7b2935ea844b04af2e22179febc0edab5a Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 12:00:52 +0200 Subject: [PATCH 91/92] Upload wheels and cli-wheels separately --- .github/workflows/release.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee9ae73cb..fd129383e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -390,12 +390,29 @@ jobs: name: ${{ github.job }}-${{ matrix.target }} path: dist - release-wheels: + release-cli-wheels: runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" needs: - cli-wheels-linux - cli-wheels-macos + steps: + - uses: actions/download-artifact@v4 + with: + pattern: "cli-wheels-*" + merge-multiple: true + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_CLI }} + with: + command: upload + args: --skip-existing * + + release-wheels: + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: - wheels-linux - wheels-macos - wheels-sdist @@ -403,7 +420,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - pattern: "*wheels-*" + pattern: "wheels-*" merge-multiple: true - name: Publish to PyPI uses: PyO3/maturin-action@v1 From 84188bdbfd6bfc1c452403725ae445c8d41fe3c3 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Sat, 29 Jun 2024 14:34:56 +0200 Subject: [PATCH 92/92] Release v0.8.0-alpha.4 --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- pineappl_capi/Cargo.toml | 2 +- pineappl_cli/Cargo.toml | 6 +++--- pineappl_py/Cargo.toml | 2 +- xtask/Cargo.toml | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c7881009..135f934f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,7 +1225,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pineappl" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.4" dependencies = [ "anyhow", "arrayvec", @@ -1250,7 +1250,7 @@ dependencies = [ [[package]] name = "pineappl_applgrid" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.4" dependencies = [ "cc", "cxx", @@ -1261,7 +1261,7 @@ dependencies = [ [[package]] name = "pineappl_capi" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.4" dependencies = [ "itertools", "pineappl", @@ -1269,7 +1269,7 @@ dependencies = [ [[package]] name = "pineappl_cli" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.4" dependencies = [ "anyhow", "assert_cmd", @@ -1300,7 +1300,7 @@ dependencies = [ [[package]] name = "pineappl_fastnlo" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.4" dependencies = [ "cxx", "cxx-build", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "pineappl_py" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.4" dependencies = [ "itertools", "ndarray", @@ -2472,7 +2472,7 @@ dependencies = [ [[package]] name = "xtask" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.4" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index 99429e62f..dab4054c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ keywords = ["high-energy-physics", "physics"] license = "GPL-3.0-or-later" repository = "https://github.com/NNPDF/pineappl" rust-version = "1.70.0" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.4" [workspace.lints.clippy] all = { level = "warn", priority = -1 } diff --git a/pineappl_capi/Cargo.toml b/pineappl_capi/Cargo.toml index ae564221f..8d3f47c47 100644 --- a/pineappl_capi/Cargo.toml +++ b/pineappl_capi/Cargo.toml @@ -16,7 +16,7 @@ version.workspace = true workspace = true [dependencies] -pineappl = { path = "../pineappl", version = "=0.8.0-alpha.3" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.4" } itertools = "0.10.1" [features] diff --git a/pineappl_cli/Cargo.toml b/pineappl_cli/Cargo.toml index 94b64c64c..fe1d19b93 100644 --- a/pineappl_cli/Cargo.toml +++ b/pineappl_cli/Cargo.toml @@ -30,9 +30,9 @@ lhapdf = { package = "managed-lhapdf", version = "0.3.2" } lz4_flex = { optional = true, version = "0.9.2" } ndarray = "0.15.4" ndarray-npy = { optional = true, version = "0.8.1" } -pineappl = { path = "../pineappl", version = "=0.8.0-alpha.3" } -pineappl_applgrid = { optional = true, path = "../pineappl_applgrid", version = "=0.8.0-alpha.3" } -pineappl_fastnlo = { optional = true, path = "../pineappl_fastnlo", version = "=0.8.0-alpha.3" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.4" } +pineappl_applgrid = { optional = true, path = "../pineappl_applgrid", version = "=0.8.0-alpha.4" } +pineappl_fastnlo = { optional = true, path = "../pineappl_fastnlo", version = "=0.8.0-alpha.4" } prettytable-rs = { default-features = false, features = ["win_crlf"], version = "0.10.0" } rayon = "1.5.1" serde = { features = ["derive"], optional = true, version = "1.0.130" } diff --git a/pineappl_py/Cargo.toml b/pineappl_py/Cargo.toml index 75a682421..177daffda 100644 --- a/pineappl_py/Cargo.toml +++ b/pineappl_py/Cargo.toml @@ -30,5 +30,5 @@ crate-type = ["cdylib"] itertools = "0.10.1" ndarray = "0.15.4" numpy = "0.20.0" -pineappl = { path = "../pineappl", version = "=0.8.0-alpha.3" } +pineappl = { path = "../pineappl", version = "=0.8.0-alpha.4" } pyo3 = { features = ["extension-module"], version = "0.20.0" } diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 1b9e53cbb..aa06a4d53 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -21,4 +21,4 @@ clap_mangen = "0.2.18" enum_dispatch = "0.3.7" #git2 = "0.17.2" #semver = "1.0.17" -pineappl_cli = { path = "../pineappl_cli", version = "=0.8.0-alpha.3" } +pineappl_cli = { path = "../pineappl_cli", version = "=0.8.0-alpha.4" }