From b6a4cee4d03aec7a2d5ea7778ed2bb51619f9f98 Mon Sep 17 00:00:00 2001
From: Tobias Stenzel <ts@flyingcircus.io>
Date: Thu, 2 Jun 2022 00:18:42 +0200
Subject: [PATCH 1/8] Working version of NixOSModule and NixContent

---
 src/batou_ext/nix.py   | 293 ++++++++++++++++++++++++++++++++++++++++-
 src/batou_ext/nixos.py |  90 +++++++++++++
 2 files changed, 377 insertions(+), 6 deletions(-)
 create mode 100644 src/batou_ext/nixos.py

diff --git a/src/batou_ext/nix.py b/src/batou_ext/nix.py
index c40926b..8dcae19 100644
--- a/src/batou_ext/nix.py
+++ b/src/batou_ext/nix.py
@@ -1,10 +1,8 @@
-import collections
-import hashlib
-import json
-import os
+import inspect
 import os.path
-import shlex
-import time
+
+from batou.component import Component, RootComponent
+from pathlib import Path
 
 import batou
 import batou.component
@@ -15,7 +13,31 @@
 import batou.lib.service
 import batou.lib.supervisor
 import batou.utils
+import collections
+import hashlib
+import json
+import os
 import pkg_resources
+import shlex
+import subprocess
+import time
+from batou import (
+    ReportingException,
+    UpdateNeeded,
+    output,
+    IPAddressConfigurationError,
+)
+from batou.environment import Environment
+from batou.host import Host
+from batou.lib.file import (
+    ManagedContentBase,
+    File,
+    Presence,
+    Mode,
+    Owner,
+    Group,
+)
+from batou.utils import NetLoc, Address
 
 
 class Package(batou.component.Component):
@@ -480,3 +502,262 @@ def update(self):
         # Start up once to load all dependencies here and not upon the first
         # use:
         self.cmd("./{} -c True".format(self.python))
+
+
+def nix_dict_to_nix(dct: dict[str, str]):
+    """Converts a dict with values that are already nixified to Nix code."""
+    content = " ".join(f"{n} = {v};" for n, v in dct.items())
+    return "{ " + content + " }"
+
+
+def seq_to_nix(seq):
+    content = " ".join(value_to_nix(v) for v in seq)
+    return "[ " + content + " ]"
+
+
+def mapping_to_nix(obj):
+    # XXX: only str keys for now
+    converted = {
+        k: conv for k, v in obj.items() if (conv := value_to_nix(v)) is not None
+    }
+    return nix_dict_to_nix(converted)
+
+
+def str_to_nix(value):
+    value = value.replace("${", "\\${")
+    return f'"{value}"'
+
+
+def environment_to_nix_dict(env: Environment):
+    dct = {
+        "base_dir": str_to_nix(env.base_dir),
+        "connect_method": str_to_nix(env.connect_method),
+        "deployment_base": str_to_nix(env.deployment_base),
+        "name": str_to_nix(env.name),
+        "target_directory": str_to_nix(env.target_directory),
+        "workdir_base": str_to_nix(env.workdir_base),
+    }
+
+    if env.host_domain is not None:
+        dct["host_domain"] = str_to_nix(env.host_domain)
+    if env.platform is not None:
+        dct["platform"] = str_to_nix(env.platform)
+    if env.service_user is not None:
+        dct["service_user"] = str_to_nix(env.service_user)
+
+    return dct
+
+
+def netloc_to_nix_dict(netloc: NetLoc):
+    return {
+        "__toString": f'_: "{netloc}"',
+        "host": str_to_nix(netloc.host),
+        "port": str(netloc.port),
+    }
+
+
+def address_to_nix_dict(addr: Address):
+    dct = {
+        "__toString": f"_: {str_to_nix(str(addr))}",
+        "connect": nix_dict_to_nix(netloc_to_nix_dict(addr.connect)),
+    }
+    try:
+        dct["listen"] = nix_dict_to_nix(netloc_to_nix_dict(addr.listen))
+    except IPAddressConfigurationError:
+        pass
+    try:
+        dct["listen_v6"] = nix_dict_to_nix(netloc_to_nix_dict(addr.listen_v6))
+    except IPAddressConfigurationError:
+        pass
+
+    return dct
+
+
+def host_to_nix_dict(host: Host):
+    return {"fqdn": str_to_nix(host.fqdn), "name": str_to_nix(host.name)}
+
+
+def value_to_nix(value):
+    if isinstance(value, str):
+        return str_to_nix(value)
+    elif isinstance(value, bool):
+        return str(value).lower()
+    elif value is None:
+        return None
+    elif isinstance(value, int):
+        return str(value)
+    elif isinstance(value, Path):
+        return str(value)
+    elif isinstance(value, dict):
+        return mapping_to_nix(value)
+    elif isinstance(value, list):
+        return seq_to_nix(value)
+    elif isinstance(value, tuple):
+        return seq_to_nix(value)
+    elif isinstance(value, Component):
+        return component_to_nix(value)
+    elif isinstance(value, Address):
+        return nix_dict_to_nix(address_to_nix_dict(value))
+    elif isinstance(value, Host):
+        return nix_dict_to_nix(host_to_nix_dict(value))
+    elif isinstance(value, Environment):
+        return nix_dict_to_nix(environment_to_nix_dict(value))
+    else:
+        raise TypeError(f"unsupported type '{type(value)}'")
+
+
+def component_to_nix(component: Component):
+    from batou_ext.nixos import NixOSModuleContext
+
+    attrs = {}
+
+    for name, value in inspect.getmembers(component):
+
+        if name.startswith("_"):
+            pass
+        elif inspect.ismethod(value) or inspect.isgenerator(value):
+            pass
+        elif name in ("sub_components", "changed"):
+            pass
+        elif isinstance(value, RootComponent):
+            if value.component is not component:
+                attrs[name] = component_to_nix(value.component)
+        elif value is component:
+            pass
+        elif isinstance(value, NixOSModuleContext):
+            pass
+        else:
+            try:
+                converted_value = value_to_nix(value)
+                if converted_value is not None:
+                    attrs[name] = converted_value
+            except TypeError as e:
+                component.log(f"Cannot convert {name}: {e.args[0]}")
+
+    return nix_dict_to_nix(attrs)
+
+
+class NixSyntaxCheckFailed(ReportingException):
+    def __init__(self, error_msg, path=None):
+        self.error_msg = error_msg.strip().removeprefix("error: ")
+        self.path = path
+
+    def __str__(self):
+        return f"Nix syntax check failed: {self.error_msg} in {self.path}"
+
+    def report(self):
+        output.error(f"Nix check {self.error_msg}")
+
+
+class NixContent(ManagedContentBase):
+
+    format_nix_code = False
+    check_nix_syntax = True
+
+    def render(self):
+        pass
+
+    def verify(self, predicting=False):
+
+        update_needed = False
+
+        if self.format_nix_code:
+            try:
+                proc = subprocess.run(
+                    ["nixfmt"],
+                    input=self.content,
+                    check=True,
+                    capture_output=True,
+                )
+                self.content = proc.stdout
+            except FileNotFoundError:
+                self.log("Cannot format Nix file, nixfmt not found.")
+            except subprocess.CalledProcessError as e:
+                self.log(f"nixfmt failed: {e.stderr}")
+
+        try:
+            super().verify(predicting)
+        except UpdateNeeded:
+            update_needed = True
+
+        if self.check_nix_syntax:
+
+            try:
+                subprocess.run(
+                    ["nix-instantiate", "--parse", "-"],
+                    input=self.content,
+                    check=True,
+                    capture_output=True,
+                )
+            except FileNotFoundError:
+                self.log(
+                    "Cannot syntax-check Nix file, nix-instantiate not found."
+                )
+            except subprocess.CalledProcessError as e:
+                raise NixSyntaxCheckFailed(
+                    e.stderr.decode("utf8"), path=self.path
+                )
+
+        if update_needed:
+            raise UpdateNeeded()
+
+
+class NixFile(File):
+
+    format_nix_code = False
+
+    def configure(self):
+        self._unmapped_path = self.path
+        self.path = self.map(self.path)
+        self += Presence(self.path, leading=self.leading)
+
+        # variation: content or source explicitly given
+
+        # The mode needs to be set early to allow batou to get out of
+        # accidental "permission denied" situations.
+        if self.mode:
+            self += Mode(self.path, mode=self.mode)
+
+        # no content or source given but file with same name
+        # exists
+        if self.content is None and not self.source:
+            guess_source = self.root.defdir + "/" + os.path.basename(self.path)
+            if os.path.isfile(guess_source):
+                self.source = guess_source
+            else:
+                # Avoid the edge case where we want to support a very simple
+                # case: specify File('asdf') and have an identical named file
+                # in the component definition directory that will be templated
+                # to the work directory.
+                #
+                # However, if you mis-spell the file, then you might
+                # accidentally end up with an empty file in the work directory.
+                # If you really want an empty File then you can either use
+                # Presence(), or (recommended) use File('asdf', content='') to
+                # make this explicit. We don't want to accidentally confuse the
+                # convenience case (template a simple file) and an edge case
+                # (have an empty file)
+                raise ValueError(
+                    "Missing implicit template file {}. Or did you want "
+                    "to create an empty file? Then use File('{}', content='').".format(
+                        guess_source, self._unmapped_path
+                    )
+                )
+        if self.content or self.source:
+
+            content = NixContent(
+                self.path,
+                source=self.source,
+                encoding=self.encoding,
+                content=self.content,
+                sensitive_data=self.sensitive_data,
+                format_nix_code=self.format_nix_code,
+            )
+            self += content
+            self.content = content.content
+
+        if self.owner:
+            self += Owner(self.path, owner=self.owner)
+
+        if self.group:
+            self += Group(self.path, group=self.group)
diff --git a/src/batou_ext/nixos.py b/src/batou_ext/nixos.py
new file mode 100644
index 0000000..1ec8b0d
--- /dev/null
+++ b/src/batou_ext/nixos.py
@@ -0,0 +1,90 @@
+from pathlib import Path
+
+from batou.component import Component
+from batou.lib.file import (
+    Purge,
+)
+
+from batou_ext.nix import (
+    nix_dict_to_nix,
+    component_to_nix,
+    NixFile,
+)
+
+# XXX: error messages stemming from the "batouModule" are displayed with
+# wrong location information, it shows the glue module instead of the actual
+# module. Can we fix that somehow?
+GLUE_MODULE_TEMPLATE = """\
+{{ pkgs, ... }}@args:
+
+with builtins;
+
+let
+  moduleArgs = (import {workdir}/{prefix}_generated_context.nix) // args;
+  batouModule = import {workdir}/{name}.nix moduleArgs;
+in 
+{{ imports = [ batouModule ]; }}
+"""
+
+
+class NixOSModuleContext(Component):
+
+    source_component: Component = None
+    prefix: str = None
+
+    def configure(self):
+
+        if self.source_component:
+            component = self.source_component
+        else:
+            component = self.parent
+
+        context = nix_dict_to_nix({"component": component_to_nix(component)})
+
+        if self.prefix is None:
+            self.prefix = component.__class__.__name__.lower()
+
+        self += NixFile(
+            f"{self.prefix}_generated_context.nix",
+            content=context,
+            format_nix_code=True,
+        )
+
+
+class NixOSModule(Component):
+
+    namevar = "name"
+
+    name: str
+    path: Path = Path("/etc/local/nixos")
+    context = None
+
+    def configure(self):
+
+        self += NixFile(f"{self.name}.nix")
+
+        if self.context is None:
+            self.context = getattr(self.parent, "nixos_context", None)
+
+        if self.context is None:
+            self.context = NixOSModuleContext(source_component=self.parent)
+            self += self.context
+
+        self += NixFile(
+            self.path / f"batou_{self.name}.nix",
+            content=GLUE_MODULE_TEMPLATE.format(
+                workdir=self.workdir, name=self.name, prefix=self.context.prefix
+            ),
+            format_nix_code=True,
+        )
+
+
+class PurgeNixOSModule(Purge):
+    namevar = "name"
+
+    name: str
+    path: Path = Path("/etc/local/nixos")
+
+    def configure(self):
+        self.pattern = self.path / f"batou_{self.name}.nix"
+        super().configure()

From 287ef73b4e5ef1d65e76e195d03d6748b1a8ad1f Mon Sep 17 00:00:00 2001
From: Tobias Stenzel <ts@flyingcircus.io>
Date: Thu, 4 Aug 2022 20:23:13 +0200
Subject: [PATCH 2/8] Create NixOSModuleContext automatically on parent if it
 is missing

---
 src/batou_ext/nixos.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/batou_ext/nixos.py b/src/batou_ext/nixos.py
index 1ec8b0d..7ea10b5 100644
--- a/src/batou_ext/nixos.py
+++ b/src/batou_ext/nixos.py
@@ -64,11 +64,12 @@ def configure(self):
         self += NixFile(f"{self.name}.nix")
 
         if self.context is None:
-            self.context = getattr(self.parent, "nixos_context", None)
-
-        if self.context is None:
-            self.context = NixOSModuleContext(source_component=self.parent)
-            self += self.context
+            if hasattr(self.parent, "nixos_context"):
+                self.context = self.parent.nixos_context
+            else:
+                self.context = NixOSModuleContext(source_component=self.parent)
+                self.parent.nixos_context = self.context
+                self.parent += self.context
 
         self += NixFile(
             self.path / f"batou_{self.name}.nix",

From 649c10d664955a20d8fcdd6aac882f57a9a19565 Mon Sep 17 00:00:00 2001
From: Tobias Stenzel <ts@flyingcircus.io>
Date: Mon, 5 Jun 2023 12:53:27 +0200
Subject: [PATCH 3/8] Fix inf recursion when converting components

---
 src/batou_ext/nix.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/batou_ext/nix.py b/src/batou_ext/nix.py
index 8dcae19..f5a64d1 100644
--- a/src/batou_ext/nix.py
+++ b/src/batou_ext/nix.py
@@ -615,15 +615,21 @@ def component_to_nix(component: Component):
 
         if name.startswith("_"):
             pass
+        elif value is component:
+            pass
         elif inspect.ismethod(value) or inspect.isgenerator(value):
             pass
         elif name in ("sub_components", "changed"):
             pass
+        elif isinstance(value, NixOSModuleContext):
+            pass
         elif isinstance(value, RootComponent):
-            if value.component is not component:
+            if value.component is not component and component.parent is not \
+                    value.component:
                 attrs[name] = component_to_nix(value.component)
-        elif value is component:
-            pass
+        elif isinstance(value, Component):
+            if value is not component.parent:
+                attrs[name] = component_to_nix(value)
         elif isinstance(value, NixOSModuleContext):
             pass
         else:

From 7dda0eb77e81fa2e44492792559945c576d532ec Mon Sep 17 00:00:00 2001
From: Christian Zagrodnick <cz@flyingcircus.io>
Date: Thu, 5 Oct 2023 08:32:31 +0200
Subject: [PATCH 4/8] fix quoting of `"`

---
 src/batou_ext/nix.py | 45 +++++++++++++++++++++-----------------------
 1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/src/batou_ext/nix.py b/src/batou_ext/nix.py
index f5a64d1..39b2895 100644
--- a/src/batou_ext/nix.py
+++ b/src/batou_ext/nix.py
@@ -1,7 +1,12 @@
+import collections
+import hashlib
 import inspect
+import json
+import os
 import os.path
-
-from batou.component import Component, RootComponent
+import shlex
+import subprocess
+import time
 from pathlib import Path
 
 import batou
@@ -13,31 +18,25 @@
 import batou.lib.service
 import batou.lib.supervisor
 import batou.utils
-import collections
-import hashlib
-import json
-import os
 import pkg_resources
-import shlex
-import subprocess
-import time
 from batou import (
+    IPAddressConfigurationError,
     ReportingException,
     UpdateNeeded,
     output,
-    IPAddressConfigurationError,
 )
+from batou.component import Component, RootComponent
 from batou.environment import Environment
 from batou.host import Host
 from batou.lib.file import (
-    ManagedContentBase,
     File,
-    Presence,
+    Group,
+    ManagedContentBase,
     Mode,
     Owner,
-    Group,
+    Presence,
 )
-from batou.utils import NetLoc, Address
+from batou.utils import Address, NetLoc
 
 
 class Package(batou.component.Component):
@@ -89,7 +88,6 @@ def namevar_for_breadcrumb(self):
 
 
 class PurgePackage(batou.component.Component):
-
     namevar = "package"
 
     def verify(self):
@@ -524,7 +522,10 @@ def mapping_to_nix(obj):
 
 
 def str_to_nix(value):
-    value = value.replace("${", "\\${")
+    # https://nixos.org/manual/nix/stable/language/values.html#type-string
+    value = (
+        value.replace("\\", "\\\\").replace("${", "\\${").replace('"', '\\"')
+    )
     return f'"{value}"'
 
 
@@ -612,7 +613,6 @@ def component_to_nix(component: Component):
     attrs = {}
 
     for name, value in inspect.getmembers(component):
-
         if name.startswith("_"):
             pass
         elif value is component:
@@ -624,8 +624,10 @@ def component_to_nix(component: Component):
         elif isinstance(value, NixOSModuleContext):
             pass
         elif isinstance(value, RootComponent):
-            if value.component is not component and component.parent is not \
-                    value.component:
+            if (
+                value.component is not component
+                and component.parent is not value.component
+            ):
                 attrs[name] = component_to_nix(value.component)
         elif isinstance(value, Component):
             if value is not component.parent:
@@ -656,7 +658,6 @@ def report(self):
 
 
 class NixContent(ManagedContentBase):
-
     format_nix_code = False
     check_nix_syntax = True
 
@@ -664,7 +665,6 @@ def render(self):
         pass
 
     def verify(self, predicting=False):
-
         update_needed = False
 
         if self.format_nix_code:
@@ -687,7 +687,6 @@ def verify(self, predicting=False):
             update_needed = True
 
         if self.check_nix_syntax:
-
             try:
                 subprocess.run(
                     ["nix-instantiate", "--parse", "-"],
@@ -709,7 +708,6 @@ def verify(self, predicting=False):
 
 
 class NixFile(File):
-
     format_nix_code = False
 
     def configure(self):
@@ -750,7 +748,6 @@ def configure(self):
                     )
                 )
         if self.content or self.source:
-
             content = NixContent(
                 self.path,
                 source=self.source,

From 6d42f439498d713113b4861a1d2eac6fae1f774d Mon Sep 17 00:00:00 2001
From: Christian Zagrodnick <cz@flyingcircus.io>
Date: Fri, 8 Dec 2023 11:18:09 +0100
Subject: [PATCH 5/8] fix/skip tests

---
 src/batou_ext/tests/test_configure.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/batou_ext/tests/test_configure.py b/src/batou_ext/tests/test_configure.py
index 7adcb42..dd9fd49 100644
--- a/src/batou_ext/tests/test_configure.py
+++ b/src/batou_ext/tests/test_configure.py
@@ -93,7 +93,7 @@ def configure(self):
         self.provide("roundcube::database", self)
 
 
-def test_prepare(root, mocker, component):
+def test_prepare(root, mocker, component, tmpdir):
     """Assert that the `prepare` method of batou_ext components can be called.
 
     The `prepare` method itself calls the `configure` method.
@@ -102,8 +102,10 @@ def test_prepare(root, mocker, component):
     args = ("namevar",) if component.namevar else ()
     required = getattr(component, "_required_params_", None)
     kw = required if required else {}
+
     instance = component(*args, **kw)
     component_name = dotted_name(component)
+
     if component_name in {
         # expecting parent component to have a `crontab` attribute:
         "batou_ext.nix.InstallCrontab",
@@ -156,5 +158,14 @@ def test_prepare(root, mocker, component):
         instance.transport_name = "my_transport_name"
         instance.graylog_host = "my_graylog_host"
         instance.graylog_port = "my_graylog_port"
-
+    elif component_name == "batou_ext.nix.NixFile":
+        instance.content = ""
+    elif component_name == "batou_ext.nixos.NixOSModuleContext":
+        instance.source_component = instance
+        root.environment.deployment_base = "/does-not-exist"
+    elif component_name == "batou_ext.nixos.NixOSModule":
+        # This cannot be used as root component and the setup is all wrong.
+        return
+        # instance.name = "mymod"
+        # (tmpdir / "mymod.nix").write_text("{}", encoding="US-ASCII")
     instance.prepare(root)

From e6ae01fdb50aa09d6a222e8b20a7b72f6a462416 Mon Sep 17 00:00:00 2001
From: Christian Zagrodnick <cz@flyingcircus.io>
Date: Fri, 8 Dec 2023 11:34:48 +0100
Subject: [PATCH 6/8] changelog

---
 CHANGES.d/20231208_113313_cz_ts_nixos_module_support.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 CHANGES.d/20231208_113313_cz_ts_nixos_module_support.md

diff --git a/CHANGES.d/20231208_113313_cz_ts_nixos_module_support.md b/CHANGES.d/20231208_113313_cz_ts_nixos_module_support.md
new file mode 100644
index 0000000..3405a67
--- /dev/null
+++ b/CHANGES.d/20231208_113313_cz_ts_nixos_module_support.md
@@ -0,0 +1 @@
+* Add `nixos.NixOSModule` to inject component attributes into .nix files.

From bace69c2a49d0be4937e81e0dc35af40c3c09302 Mon Sep 17 00:00:00 2001
From: Christian Zagrodnick <cz@flyingcircus.io>
Date: Fri, 8 Dec 2023 11:39:26 +0100
Subject: [PATCH 7/8] coding style

---
 src/batou_ext/nixos.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/batou_ext/nixos.py b/src/batou_ext/nixos.py
index 7ea10b5..c1294f9 100644
--- a/src/batou_ext/nixos.py
+++ b/src/batou_ext/nixos.py
@@ -1,15 +1,9 @@
 from pathlib import Path
 
 from batou.component import Component
-from batou.lib.file import (
-    Purge,
-)
+from batou.lib.file import Purge
 
-from batou_ext.nix import (
-    nix_dict_to_nix,
-    component_to_nix,
-    NixFile,
-)
+from batou_ext.nix import NixFile, component_to_nix, nix_dict_to_nix
 
 # XXX: error messages stemming from the "batouModule" are displayed with
 # wrong location information, it shows the glue module instead of the actual
@@ -22,7 +16,7 @@
 let
   moduleArgs = (import {workdir}/{prefix}_generated_context.nix) // args;
   batouModule = import {workdir}/{name}.nix moduleArgs;
-in 
+in
 {{ imports = [ batouModule ]; }}
 """
 

From d7f47725613aeb9786cf44de840a4d84406040b1 Mon Sep 17 00:00:00 2001
From: Christian Zagrodnick <cz@flyingcircus.io>
Date: Fri, 8 Dec 2023 13:20:20 +0100
Subject: [PATCH 8/8] python3.6 compatibility

---
 src/batou_ext/nix.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/batou_ext/nix.py b/src/batou_ext/nix.py
index 39b2895..180e7bc 100644
--- a/src/batou_ext/nix.py
+++ b/src/batou_ext/nix.py
@@ -502,7 +502,7 @@ def update(self):
         self.cmd("./{} -c True".format(self.python))
 
 
-def nix_dict_to_nix(dct: dict[str, str]):
+def nix_dict_to_nix(dct):
     """Converts a dict with values that are already nixified to Nix code."""
     content = " ".join(f"{n} = {v};" for n, v in dct.items())
     return "{ " + content + " }"
@@ -515,9 +515,12 @@ def seq_to_nix(seq):
 
 def mapping_to_nix(obj):
     # XXX: only str keys for now
-    converted = {
-        k: conv for k, v in obj.items() if (conv := value_to_nix(v)) is not None
-    }
+
+    converted = {}
+    for k, v in obj.items():
+        conv = value_to_nix(v)
+        if conv is not None:
+            converted[k] = conv
     return nix_dict_to_nix(converted)