Improvements for ioc-dlslinuxvac

.copier-answers.yml

@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2.5.0
+_commit: 2.6.0
 _src_path: gh:diamondlightsource/python-copier-template
 author_email: [email protected]
 author_name: Giles Knap

.devcontainer/devcontainer.json

@@ -28,11 +28,11 @@
         }
     },
     "features": {
-        // Some default things like git config
-        "ghcr.io/devcontainers/features/common-utils:2": {
-            "upgradePackages": false
-        }
+        // add in eternal history and other bash features
+        "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1.0.0": {}
     },
+    // Create the config folder for the bash-config feature
+    "initializeCommand": "mkdir -p ${localEnv:HOME}/.config/bash-config",
     "runArgs": [
         // Allow the container to access the host X11 display and EPICS CA
         "--net=host",

.github/CONTRIBUTING.md

@@ -24,4 +24,4 @@ It is recommended that developers use a [vscode devcontainer](https://code.visua
 
 This project was created using the [Diamond Light Source Copier Template](https://github.com/DiamondLightSource/python-copier-template) for Python projects.
 
-For more information on common tasks like setting up a developer environment, running the tests, and setting a pre-commit hook, see the template's [How-to guides](https://diamondlightsource.github.io/python-copier-template/2.5.0/how-to.html).
+For more information on common tasks like setting up a developer environment, running the tests, and setting a pre-commit hook, see the template's [How-to guides](https://diamondlightsource.github.io/python-copier-template/2.6.0/how-to.html).

.vscode/settings.json

@@ -9,4 +9,7 @@
     "[python]": {
         "editor.defaultFormatter": "charliermarsh.ruff",
     },
+    "python.testing.pytestArgs": [
+        "tests"
+    ],
 }

README.md

@@ -5,19 +5,26 @@
 
 # builder2ibek
 
-Conversion tool for DLS XML builder IOC instances to ibek ioc.yaml
-
-This is where you should write a short paragraph that describes what your module does,
-how it does it, and why people should use it.
+A tool suite for converting DLS XML builder projects to epics-containers ibek.
 
 Source          | <https://github.com/epics-containers/builder2ibek>
 :---:           | :---:
 PyPI            | `pip install builder2ibek`
 Releases        | <https://github.com/epics-containers/builder2ibek/releases>
 
-This is where you should put some images or code snippets that illustrate
-some relevant examples. If it is a library then you might put some
-introductory code here:
+<pre><font color="#AAAAAA">╭─ Commands ───────────────────────────────────────────────────────────────────╮</font>
+<font color="#AAAAAA">│ </font><font color="#2AA1B3"><b>xml2yaml       </b></font> Convert a builder XML IOC instance definition file into an   │
+<font color="#AAAAAA">│ </font><font color="#2AA1B3"><b>               </b></font> ibek YAML file                                               │
+<font color="#AAAAAA">│ </font><font color="#2AA1B3"><b>beamline2yaml  </b></font> Convert all IOCs in a BLXXI-SUPPORT project into a set of    │
+<font color="#AAAAAA">│ </font><font color="#2AA1B3"><b>               </b></font> ibek services folders (TODO)                                 │
+<font color="#AAAAAA">│ </font><font color="#2AA1B3"><b>autosave       </b></font> Convert DLS autosave DB template comments into autosave req  │
+<font color="#AAAAAA">│ </font><font color="#2AA1B3"><b>               </b></font> files                                                        │
+<font color="#AAAAAA">│ </font><font color="#2AA1B3"><b>db-compare     </b></font> Compare two DB files and output the differences              │
+<font color="#AAAAAA">╰──────────────────────────────────────────────────────────────────────────────╯</font>
+</pre>
+
+```bash
+builder2ibek
 
 ```python
 from builder2ibek import __version__

src/builder2ibek/__main__.py

@@ -1,16 +1,14 @@
-import re
 from pathlib import Path
 from typing import Optional
 
 import typer
-from ruamel.yaml import YAML, CommentedMap
 
 from builder2ibek import __version__
-from builder2ibek.builder import Builder
-from builder2ibek.convert import dispatch
+from builder2ibek.convert import convert_file
+from builder2ibek.db2autosave import parse_templates
+from builder2ibek.dbcompare import compare_dbs
 
 cli = typer.Typer()
-yaml = YAML()
 
 
 def version_callback(value: bool):
@@ -29,64 +27,71 @@ def main(
         help="Print the version of builder2ibek and exit",
     ),
 ):
-    """Convert build XML to ibek YAML"""
+    """Convert xmlbuilder assets to epics-containers assets"""
 
 
 @cli.command()
-def file(
+def xml2yaml(
     xml: Path = typer.Argument(..., help="Filename of the builder XML file"),
     yaml: Optional[Path] = typer.Option(..., help="Output file"),
-    schema: Optional[str] = typer.Option(
+    schema: str = typer.Option(
         "/epics/ibek-defs/ioc.schema.json",
         help="Generic IOC schema (added to top of the yaml output)",
     ),
 ):
-    def tidy_up(yaml):
-        # add blank lines between major fields
-        for field in [
-            "ioc_name",
-            "description",
-            "entities",
-            "  - type",
-        ]:
-            yaml = re.sub(rf"(\n{field})", "\n\\g<1>", yaml)
-        return yaml
-
-    """Convert a single builder XML file into a single ibek YAML"""
-    builder = Builder()
-    builder.load(xml)
-    ioc = dispatch(builder, xml)
-
     if not yaml:
         yaml = xml.absolute().with_suffix("yaml")
 
-    ruamel = YAML()
-
-    ruamel.default_flow_style = False
-    # this attribute is for internal use, remove before serialising
-    delattr(ioc, "source_file")
-    yaml_map = CommentedMap(ioc.model_dump())
-
-    # add support yaml schema
-    yaml_map.yaml_add_eol_comment(f"yaml-language-server: $schema={schema}", column=0)
-
-    ruamel.indent(mapping=2, sequence=4, offset=2)
-
-    with yaml.open("w") as stream:
-        ruamel.dump(yaml_map, stream, transform=tidy_up)
+    convert_file(xml, yaml, schema)
 
 
 @cli.command()
-def beamline(
+def beamline2yaml(
     input: Path = typer.Argument(..., help="Path to root folder BLXX-BUILDER"),
     output: Path = typer.Argument(..., help="Output root folder"),
 ):
     """
-    Convert a beamline's IOCs from builder to ibek
+    <<<<<<< HEAD
+        Convert whole beamline's IOCs from builder to ibek (TODO)
+    =======
+        Convert all IOCs in a BLXXI-SUPPORT project into a set of ibek services
+        folders (TODO)
+    >>>>>>> 4af36b9 (add db compare)
     """
     typer.echo("Not implemented yet")
     raise typer.Exit(code=1)
 
 
+@cli.command()
+def autosave(
+    out_folder: Path = typer.Option(
+        ".", help="Output folder to write autosave request files"
+    ),
+    db_list: list[Path] = typer.Argument(
+        ..., help="List of DB templates with autosave comments"
+    ),
+):
+    """
+    Convert DLS autosave DB template comments into autosave req files
+    """
+    parse_templates(out_folder, db_list)
+
+
+@cli.command()
+def db_compare(
+    original: Path,
+    new: Path,
+    ignore: list[str] = typer.Option(
+        [], help="List of record name sub strings to ignore"
+    ),
+    output: Optional[Path] = typer.Option(None, help="Output file"),
+):
+    """
+    Compare two DB files and output the differences
+    """
+
+    compare_dbs(original, new, ignore, output)
+
+
 if __name__ == "__main__":
     cli()

src/builder2ibek/convert.py

@@ -2,20 +2,57 @@
 Generic XML to YAML conversion functions
 """
 
+import re
+from pathlib import Path
 from typing import Any
 
+from ruamel.yaml import YAML, CommentedMap
+
 from builder2ibek.builder import Builder, Element
 from builder2ibek.moduleinfos import module_infos
 from builder2ibek.types import Entity, Generic_IOC
 
 
+def convert_file(xml: Path, yaml: Path, schema: str):
+    def tidy_up(yaml):
+        # add blank lines between major fields
+        for field in [
+            "ioc_name",
+            "description",
+            "entities",
+            "  - type",
+        ]:
+            yaml = re.sub(rf"(\n{field})", "\n\\g<1>", yaml)
+        return yaml
+
+    """Convert a single builder XML file into a single ibek YAML"""
+    builder = Builder()
+    builder.load(xml)
+    ioc = dispatch(builder, xml)
+
+    ruamel = YAML()
+
+    ruamel.default_flow_style = False
+    # this attribute is for internal use, remove before serialising
+    delattr(ioc, "source_file")
+    yaml_map = CommentedMap(ioc.model_dump())
+
+    # add support yaml schema
+    yaml_map.yaml_add_eol_comment(f"yaml-language-server: $schema={schema}", column=0)
+
+    ruamel.indent(mapping=2, sequence=4, offset=2)
+
+    with yaml.open("w") as stream:
+        ruamel.dump(yaml_map, stream, transform=tidy_up)
+
+
 def dispatch(builder: Builder, filename) -> Generic_IOC:
     """
     Dispatch every element in the XML to the correct convertor
     and build a generic IOC from the converted Entities
     """
     ioc = Generic_IOC(
-        ioc_name="{{  __utils__.get_env('IOC_NAME') }}",
+        ioc_name="{{  _global.get_env('IOC_NAME') }}",
         description="auto-generated by https://github.com/epics-containers/builder2ibek",
         # some default entities for all IOC instances
         entities=[
@@ -51,11 +88,11 @@ def do_one_element(element: Element, ioc: Generic_IOC):
     # then dispatch to a specific handler if there is one
     assert isinstance(element, Element)
 
-    info = (
-        module_infos[element.module]
-        if element.module in module_infos
-        else module_infos["generic"]
-    )
+    if element.module in module_infos:
+        info = module_infos[element.module]
+    else:
+        info = module_infos["generic"]
+        info.yaml_component = element.module
 
     entity.type = f"{info.yaml_component}.{element.name}"
     new_xml = info.handler(entity, element.name, ioc)

src/builder2ibek/converters/autosave.py

@@ -10,5 +10,12 @@ def handler(entity: Entity, entity_type: str, ioc: Generic_IOC):
     XML to YAML specialist convertor function for the pvlogging support module
     """
 
-    # TODO not supporting this yet - just remove it
-    entity.delete_me()
+    if entity_type == "Autosave":
+        entity.rename("iocName", "P")
+        entity.P += ":"
+        # TODO if this is a motor then set entity.postions_req_period = 5
+        entity.remove("bl")
+        entity.remove("ip")
+        entity.remove("name")
+        entity.remove("path")
+        entity.remove("server")

src/builder2ibek/converters/terminalserver.py

@@ -0,0 +1,17 @@
+from builder2ibek.converters.globalHandler import globalHandler
+from builder2ibek.types import Entity, Generic_IOC
+
+xml_component = "terminalServer"
+
+
+@globalHandler
+def handler(entity: Entity, entity_type: str, ioc: Generic_IOC):
+    """
+    XML to YAML specialist convertor function for the terminalServer module
+    """
+
+    if entity_type == "Moxa":
+        if entity.NCHANS > 16:
+            entity.type = "terminalServer.Moxa32"
+        else:
+            entity.type = "terminalServer.Moxa16"

src/builder2ibek/db2autosave.py

@@ -0,0 +1,43 @@
+import re
+from pathlib import Path
+
+regex_autosave = [
+    re.compile(rf'# *% *autosave *{n} *(.*)[\s\S]*?record *\(.*, *"?([^"]*)"?\)')
+    for n in range(3)
+]
+
+
+def parse_templates(out_folder: Path, db_list: list[Path]):
+    """
+    DLS has 3 autosave levels
+        0 = save on pass 0
+        1 = save on pass 0 and 1
+        2 = save on pass 1 only
+
+    Areadetector and other support modules use:
+        template_name_positions.req for save on pass 0
+        template_name_settings.req for save on pass 0 and pass 1
+        the autosave docs seem to back the idea that pass 1 only is never used
+
+    So this translation will make
+        0 => template_name_positions.req
+        1 => template_name_settings.req
+    """
+    for db in db_list:
+        text = db.read_text()
+
+        positions: set[str] = set()
+        settings: set[str] = set()
+        for n in range(3):
+            match n:
+                case 0:
+                    this_set = positions
+                case 1 | 2:
+                    this_set = settings
+            for result in regex_autosave[n].finditer(text):
+                this_set.add(f"{result.group(2)} {result.group(1)}")
+
+        req_file = out_folder / f"{db.stem}_positions.req"
+        req_file.write_text("\n".join(positions) + "\n")
+        req_file = out_folder / f"{db.stem}_settings.req"
+        req_file.write_text("\n".join(settings) + "\n")