nix-flake: Clean up python packages

* Use overlay to add extra modules to the package set * Properly set dependencies for package derivations * Package missing dependencies * Upgrade packaged sphinx-tsn-theme version to 2024.1.0 * Slightly prettify python-dependencies.nix Signed-off-by: Sarah Renkhoff <[email protected]>
tiacsys · Feb 1, 2024 · 7b3be57 · 7b3be57
1 parent ab2fbc8
commit 7b3be57
Show file tree

Hide file tree

Showing 11 changed files with 215 additions and 88 deletions.
diff --git a/flake.nix b/flake.nix
@@ -19,22 +19,16 @@
     let
       pkgs = import nixpkgs {
         system = "x86_64-linux";
+
+        overlays = [
+          (import ./nix/overlays/extra-packages.nix)
+        ];
       };
+
       zephyr-sdk-pkg = zephyr-sdk.packages.x86_64-linux.zephyr-sdk-0-16-4;
       pypkgs = pkgs.python311Packages;
       workspace = get-flake ./nix/workspace;
 
-      # Extend upstream packages by our own ones
-      extra-pypkgs = {
-        pymcuprog = pypkgs.callPackage ./nix/pymcuprog/default.nix { };
-        pykitinfo = pypkgs.callPackage ./nix/pykitinfo/default.nix { };
-        sphinx-csv-filter = pypkgs.callPackage ./nix/sphinx-csv-filter/default.nix { };
-        sphinx-tsn-theme = pypkgs.callPackage ./nix/sphinx-tsn-theme/default.nix { };
-        sphinxcontrib-svg2pdfconverter = pypkgs.callPackage ./nix/sphinxcontrib-svg2pdfconverter/default.nix { };
-        clang-format = pypkgs.callPackage ./nix/clang-format/default.nix { };
-      };
-      pypkgs-extended = pypkgs // extra-pypkgs;
-
       bridle-deps = [
         # Zephyr SDK
         zephyr-sdk-pkg
@@ -57,8 +51,14 @@
         pkgs.gitlint
         pkgs.clang-tools
 
-        # Python with packages that need to be available for import
-        (pkgs.python311.withPackages (ps: with ps; (import ./nix/python-dependencies.nix { pypkgs = pypkgs-extended; })))
+        # Python with packages that need to be available for import Use lower
+        # level buildEnv instead of withPackages because we need access to
+        # ignoreCollisions
+        (pkgs.python3.buildEnv.override {
+          extraLibs = (import ./nix/python-dependencies.nix { inherit pypkgs; });
+          # pykitinfo has file collisions with pyedbglib
+          ignoreCollisions = true;
+        })
       ];
 
     in

diff --git a/nix/overlays/extra-packages.nix b/nix/overlays/extra-packages.nix
@@ -0,0 +1,16 @@
+final: prev: {
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (
+      python-final: python-prev: {
+        pyedbglib = prev.python3Packages.callPackage ../pyedbglib/default.nix { };
+        pydebuggerconfig = prev.python3Packages.callPackage ../pydebuggerconfig/default.nix { };
+        pykitinfo = prev.python3Packages.callPackage ../pykitinfo/default.nix { };
+        pymcuprog = prev.python3Packages.callPackage ../pymcuprog/default.nix { };
+        sphinx-csv-filter = prev.python3Packages.callPackage ../sphinx-csv-filter/default.nix { };
+        sphinx-tsn-theme = prev.python3Packages.callPackage ../sphinx-tsn-theme/default.nix { };
+        sphinxcontrib-svg2pdfconverter = prev.python3Packages.callPackage ../sphinxcontrib-svg2pdfconverter/default.nix { };
+        clang-format = prev.python3Packages.callPackage ../clang-format/default.nix { };
+      }
+    )
+  ];
+}
diff --git a/nix/pydebuggerconfig/default.nix b/nix/pydebuggerconfig/default.nix
@@ -0,0 +1,25 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, pyedbglib
+}:
+
+buildPythonPackage rec {
+
+  pname = "pydebuggerconfig";
+  version = "3.9.2.121";
+
+  src = fetchPypi {
+    inherit pname version;
+    format = "wheel";
+    python = "py3";
+    dist = "py3";
+    sha256 = "sha256-SrGDaWW0Or8AFwb148xOGfKQuTv3ZoacU3rZFrk8tvM=";
+  };
+
+  format = "wheel";
+
+  propagatedBuildInputs = [
+    pyedbglib
+  ];
+}
diff --git a/nix/pyedbglib/default.nix b/nix/pyedbglib/default.nix
@@ -0,0 +1,25 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, pyserial
+}:
+
+buildPythonPackage rec {
+
+  pname = "pyedbglib";
+  version = "2.23.0.14";
+
+  src = fetchPypi {
+    inherit pname version;
+    format = "wheel";
+    python = "py3";
+    dist = "py3";
+    sha256 = "sha256-NOqJfS4Xk9lw5KWk99YVOcrW89slXYLFQUlTI8fiv+M=";
+  };
+
+  format = "wheel";
+
+  propagatedBuildInputs = [
+    pyserial
+  ];
+}
diff --git a/nix/pykitinfo/default.nix b/nix/pykitinfo/default.nix
@@ -1,4 +1,12 @@
-{ lib, fetchPypi, buildPythonPackage }:
+{ lib
+, fetchPypi
+, buildPythonPackage
+, pyusb
+, pyyaml
+, appdirs
+, pyedbglib
+, pydebuggerconfig
+}:
 
 buildPythonPackage rec {
 
@@ -14,4 +22,12 @@ buildPythonPackage rec {
   };
 
   format = "wheel";
+
+  propagatedBuildInputs = [
+    pyusb
+    pyyaml
+    appdirs
+    pyedbglib
+    pydebuggerconfig
+  ];
 }
diff --git a/nix/pymcuprog/default.nix b/nix/pymcuprog/default.nix
@@ -1,4 +1,8 @@
-{ lib, fetchPypi, buildPythonPackage }:
+{ lib
+, fetchPypi
+, buildPythonPackage
+, pyedbglib
+}:
 
 buildPythonPackage rec {
 
@@ -9,8 +13,13 @@ buildPythonPackage rec {
     inherit pname version;
     format = "wheel";
     python = "py3";
+    dist = "py3";
     sha256 = "sha256-Yqkiww2mdL5BRyCO6IjbazCRIK2OLKQoQe0ooZmEhlk=";
   };
 
   format = "wheel";
+
+  propagatedBuildInputs = [
+    pyedbglib
+  ];
 }
diff --git a/nix/python-dependencies.nix b/nix/python-dependencies.nix
@@ -1,61 +1,61 @@
-{pypkgs}: [
-  pypkgs.anytree
-  pypkgs.breathe
-  pypkgs.canopen
-  pypkgs.cbor
-  pypkgs.colorama
-  pypkgs.commonmark
-  pypkgs.coverage
-  pypkgs.docutils
-  pypkgs.ecdsa
-  pypkgs.gcovr
-  pypkgs.graphviz
-  pypkgs.grpcio-tools
-  pypkgs.imagesize
-  pypkgs.imgtool
-  pypkgs.intelhex
-  pypkgs.junit2html
-  pypkgs.junitparser
-  pypkgs.lpc-checksum
-  pypkgs.lxml
-  pypkgs.mock
-  pypkgs.mypy
-  pypkgs.natsort
-  pypkgs.packaging
-  pypkgs.parse
-  pypkgs.pillow
-  pypkgs.ply
-  pypkgs.progress
-  pypkgs.protobuf
-  pypkgs.psutil
-  pypkgs.pyelftools
-  pypkgs.pygithub
-  pypkgs.pygments
-  pypkgs.pykitinfo
-  pypkgs.pykwalify
-  pypkgs.pylink-square
-  pypkgs.pylint
-  pypkgs.pymcuprog
-  pypkgs.pyocd
-  pypkgs.pyserial
-  pypkgs.pytest
-  pypkgs.python-magic
-  pypkgs.pyyaml
-  pypkgs.recommonmark
-  pypkgs.regex
-  pypkgs.requests
-  pypkgs.sphinx
-  pypkgs.sphinx-copybutton
-  pypkgs.sphinx-csv-filter
-  pypkgs.sphinx-notfound-page
-  pypkgs.sphinx-rtd-theme
-  pypkgs.sphinx-tabs
-  pypkgs.sphinx-togglebutton
-  pypkgs.sphinx-tsn-theme
-  pypkgs.sphinxcontrib-mscgen
-  pypkgs.sphinxcontrib-svg2pdfconverter
-  pypkgs.tabulate
-  pypkgs.west
-  pypkgs.yamllint
-  pypkgs.zcbor
+{ pypkgs }: with pypkgs; [
+anytree
+breathe
+canopen
+cbor
+colorama
+commonmark
+coverage
+docutils
+ecdsa
+gcovr
+graphviz
+grpcio-tools
+imagesize
+imgtool
+intelhex
+junit2html
+junitparser
+lpc-checksum
+lxml
+mock
+mypy
+natsort
+packaging
+parse
+pillow
+ply
+progress
+protobuf
+psutil
+pyelftools
+pygithub
+pygments
+pykitinfo
+pykwalify
+pylink-square
+pylint
+pymcuprog
+pyocd
+pyserial
+pytest
+python-magic
+pyyaml
+recommonmark
+regex
+requests
+sphinx
+sphinx-copybutton
+sphinx-csv-filter
+sphinx-notfound-page
+sphinx-rtd-theme
+sphinx-tabs
+sphinx-togglebutton
+sphinx-tsn-theme
+sphinxcontrib-mscgen
+sphinxcontrib-svg2pdfconverter
+tabulate
+west
+yamllint
+zcbor
 ]
diff --git a/nix/sphinx-csv-filter/default.nix b/nix/sphinx-csv-filter/default.nix
@@ -1,4 +1,11 @@
-{ lib, fetchPypi, buildPythonPackage, setuptools }:
+{ lib
+, fetchPypi
+, buildPythonPackage
+, pythonRelaxDepsHook
+, setuptools
+, docutils
+, sphinx
+}:
 
 buildPythonPackage rec {
 
@@ -13,5 +20,16 @@ buildPythonPackage rec {
   format = "pyproject";
   nativeBuildInputs = [
     setuptools
+    pythonRelaxDepsHook
+  ];
+
+  propagatedBuildInputs = [
+    docutils
+    sphinx
+  ];
+
+  # package isn't actually broken with docutils >=0.19
+  pythonRelaxDeps = [
+    "docutils"
   ];
 }
diff --git a/nix/sphinx-tsn-theme/default.nix b/nix/sphinx-tsn-theme/default.nix
@@ -1,19 +1,27 @@
-{ lib, buildPythonPackage, fetchFromGitHub, setuptools }:
+{ lib
+, buildPythonPackage
+, fetchPypi
+, setuptools
+, sphinx-rtd-theme
+}:
 
 buildPythonPackage rec {
 
   pname = "sphinx-tsn-theme";
-  version = "2023.12.1";
+  version = "2024.1.0";
 
-  src = fetchFromGitHub {
-    owner = "tiacsys";
-    repo = "sphinx-tsn-theme";
-    rev = "2023.12.1";
-    hash = "sha256-sj08Q7v79YYq5i/uIEdLXLFUmGg4x+QnJfwLyGr20nU=";
+  src = fetchPypi {
+    pname = "sphinx_tsn_theme";
+    inherit version;
+    format = "wheel";
+    python = "py3";
+    dist = "py3";
+    hash = "sha256-2BECC7XvezBzUk/Zs3CDfEfgX7K4YqcCYvsUQBLw0qs=";
   };
 
-  format = "pyproject";
-  nativeBuildInputs = [
-    setuptools
+  format = "wheel";
+
+  propagatedBuildInputs = [
+    sphinx-rtd-theme
   ];
 }
diff --git a/nix/sphinxcontrib-svg2pdfconverter/default.nix b/nix/sphinxcontrib-svg2pdfconverter/default.nix
@@ -1,4 +1,9 @@
-{ lib, fetchFromGitHub, buildPythonPackage, setuptools }:
+{ lib
+, fetchFromGitHub
+, buildPythonPackage
+, setuptools
+, sphinx
+}:
 
 buildPythonPackage rec {
 
@@ -13,7 +18,12 @@ buildPythonPackage rec {
   };
 
   format = "pyproject";
+
   nativeBuildInputs = [
     setuptools
   ];
+
+  propagatedBuildInputs = [
+    sphinx
+  ];
 }
diff --git a/scripts/west_commands/nix_update_python.py b/scripts/west_commands/nix_update_python.py
@@ -59,7 +59,7 @@ def filter_special_cases(names: str):
 
 
 def nix_list(entries: [str]):
-    return """{pypkgs}: [\n""" + '\n'.join(["  pypkgs." + entry for entry in entries]) + "\n]"
+    return """{ pypkgs }: with pypkgs; [\n""" + '\n'.join(entries) + "\n]"
 
 def generate_nix_from_paths(paths: [str]):
     # Parse into a list of package names