Move permanent part of fkutils here

README.md

@@ -1,4 +1,3 @@
 # Pineko
 
 This repository contains specifications for the apfelcomb code replacement.
-

data/ekos/.gitignore

@@ -0,0 +1 @@
+*.tar

data/fktables/.gitignore

@@ -0,0 +1 @@
+*.pineappl.lz4

data/grids/.gitignore

@@ -0,0 +1 @@
+*.pineappl.lz4

data/grids/common/.gitignore

@@ -0,0 +1 @@
+*.pineappl.lz4

data/grids/load.sh

@@ -0,0 +1,8 @@
+# copy from remote
+
+# common example
+# scp dom.mi.infn.it:/media/FK/fktables/data/grids/200-LHCB_DY_13TEV_DIELECTRON.pineappl.lz4 ./common/LHCB_DY_13TEV_DIELECTRON.pineappl.lz4
+
+# theory example
+# scp dom.mi.infn.it:/media/FK/fktables/data/grids/200-SLAC_NC_EM_D_F2.pineappl.lz4 ./200/SLAC_NC_EM_D_F2.pineappl.lz4
+# scp dom.mi.infn.it:/media/FK/fktables/data/grids/200-SLAC_NC_EM_P_F2.pineappl.lz4 ./200/SLAC_NC_EM_P_F2.pineappl.lz4

data/operator_cards/.gitignore

@@ -0,0 +1 @@
+*.yaml

data/theory_cards/.gitignore

@@ -0,0 +1 @@
+*.yaml

data/ymldb/.gitignore

@@ -0,0 +1 @@
+*.yaml

data/ymldb/load.sh

@@ -0,0 +1,2 @@
+# copy from remote
+# scp dom.mi.infn.it:/media/FK/fktables/data/fktocommon/*.yaml .

logs/.gitignore

@@ -0,0 +1 @@
+*.log

pineko.toml

@@ -0,0 +1,12 @@
+[paths]
+ymldb = "data/ymldb"
+grids = "data/grids"
+grids_common = "data/grids/common"
+operator_cards = "data/operator_cards"
+theory_cards = "data/theory_cards"
+opcard_template = "data/operator_cards/_template.yaml"
+ekos = "data/ekos"
+fktables = "data/fktables"
+
+[paths.logs]
+eko = "logs/eko"

pyproject.toml

@@ -27,8 +27,6 @@ tomli = "^2.0.1"
 
 [tool.poetry.dev-dependencies]
 # code review
-black = "^21.6b0"
-isort = "^5.9.2"
 pylint = "^2.9.3"
 # debug / explore
 pdbpp = "^0.10.3"
@@ -43,11 +41,8 @@ pineko = "pineko:command"
 
 [tool.poe.tasks]
 test = "pytest tests"
-lint = "pylint src/*/*.py -E"
-lint-warnings = "pylint src/*/*.py --exit-zero"
-black = "black ."
-isort = "isort ."
-format = ["black", "isort"]
+lint = "pylint src/**/*.py -E"
+lint-warnings = "pylint src/**/*.py --exit-zero"
 pineko = "pineko"
 
 [tool.poetry-dynamic-versioning]

run.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
-
 # -*- coding: utf-8 -*-
+
 import logging
 import pathlib
 import sys

run2.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from pineko import configs
 
 configs.configs = configs.defaults(configs.load())

src/pineko/__main__.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
 from . import command
 
-command()
+command()  # pylint: disable=no-value-for-parameter
+# see https://stackoverflow.com/questions/49680191/click-and-pylint

src/pineko/cli/__init__.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
-from . import check, compare, convolute, opcard
+from . import check, compare, convolute, opcard, theory_ekos, theory_fks, theory_opcards
 from ._base import command

src/pineko/cli/opcard.py

@@ -1,25 +1,26 @@
 # -*- coding: utf-8 -*-
 import click
-import pineappl
 import rich
-import yaml
 
 from .. import evolve
 from ._base import command
 
 
 @command.command("opcard")
 @click.argument("pineappl_path", metavar="PINEAPPL", type=click.Path(exists=True))
-@click.argument("default_card_path", metavar="DEFAULT_CARD", type=click.Path(exists=True))
+@click.argument(
+    "default_card_path", metavar="DEFAULT_CARD", type=click.Path(exists=True)
+)
 @click.argument("opcard_path", metavar="OPCARD", type=click.Path())
 def subcommand(pineappl_path, default_card_path, opcard_path):
     """Write EKO card for PineAPPL grid.
 
     Writes a copy of the default card from DEFAULT_CARD to OPCARD
     with the adjusted x grid and Q2 grid read from PINEAPPL.
     """
-    pineappl_grid = pineappl.grid.Grid.read(pineappl_path)
-    with open(default_card_path, "r", encoding="UTF-8") as f:
-        default_card = yaml.safe_load(f)
-    _x_grid,q2_grid = evolve.write_operator_card(pineappl_grid, default_card, opcard_path)
-    rich.print(f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}")
+    _x_grid, q2_grid = evolve.write_operator_card_from_file(
+        pineappl_path, default_card_path, opcard_path
+    )
+    rich.print(
+        f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}"
+    )

src/pineko/cli/theory_ekos.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+import logging
+
+import click
+import eko
+import rich
+import yaml
+
+from .. import configs, parser, theory_card
+from ._base import command
+
+
+@command.command("theory_ekos")
+@click.argument("theory_id", type=click.INT)
+@click.argument("datasets", type=click.STRING, nargs=-1)
+@click.option("--logs", is_flag=True, help="dump logs")
+def subcommand(theory_id, datasets, logs):
+    """Compute EKOs for all FK tables in all datasets."""
+    # setup data
+    paths = configs.configs["paths"]
+    tcard = theory_card.load(theory_id)
+    eko_path = paths["ekos"] / str(theory_id)
+    eko_path.mkdir(exist_ok=True)
+    # iterate datasets
+    for ds in datasets:
+        rich.print(f"Analyze {ds}")
+        # iterate grids
+        grids = parser.load_grids(theory_id, ds)
+        for name in grids.keys():
+            opcard_path = paths["operator_cards"] / f"{name}.yaml"
+            with open(opcard_path, encoding="utf-8") as f:
+                ocard = yaml.safe_load(f)
+            eko_filename = eko_path / f"{name}.tar"
+            # activate logging
+            if logs and paths["logs"]["eko"]:
+                log_path = paths["logs"]["eko"] / f"{theory_id}-{ds}-{name}.log"
+                logStdout = logging.FileHandler(log_path)
+                logStdout.setLevel(logging.INFO)
+                logStdout.setFormatter(logging.Formatter("%(message)s"))
+                logging.getLogger("eko").handlers = []
+                logging.getLogger("eko").addHandler(logStdout)
+                logging.getLogger("eko").setLevel(logging.INFO)
+            # do it!
+            ops = eko.run_dglap(theory_card=tcard, operators_card=ocard)
+            ops.dump_tar(eko_filename)
+            rich.print(f"[green]Success:[/] Wrote EKO to {eko_filename}")
+        print()

src/pineko/cli/theory_fks.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+import click
+import rich
+
+from .. import configs, evolve, parser, theory_card
+from ._base import command
+
+
+@command.command("theory_fks")
+@click.argument("theory_id", type=click.INT)
+@click.argument("datasets", type=click.STRING, nargs=-1)
+@click.option("--logs", is_flag=True, help="dump logs")
+def subcommand(theory_id, datasets, logs):
+    """Compute FK tables in all datasets."""
+    # setup data
+    paths = configs.configs["paths"]
+    tcard = theory_card.load(theory_id)
+    eko_path = paths["ekos"] / str(theory_id)
+    fk_path = paths["fktables"] / str(theory_id)
+    fk_path.mkdir(exist_ok=True)
+    # iterate datasets
+    for ds in datasets:
+        rich.print(f"Analyze {ds}")
+        # iterate grids
+        grids = parser.load_grids(theory_id, ds)
+        for name, grid in grids.items():
+            eko_filename = eko_path / f"{name}.tar"
+            fk_filename = fk_path / f"{ds}-{name}.{parser.ext}"
+            # activate logging
+            # if logs and paths["logs"]["eko"]:
+            evolve.evolve_grid(
+                grid, eko_filename, fk_filename, 1 + int(tcard["PTO"]), 0
+            )
+            # do it!
+            rich.print(f"[green]Success:[/] Wrote FK table to {fk_filename}")
+        print()

src/pineko/cli/theory_opcards.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+import click
+import rich
+
+from .. import configs, evolve, parser
+from ._base import command
+
+
+@command.command("theory_opcards")
+@click.argument("theory_id", type=click.INT)
+@click.argument("datasets", type=click.STRING, nargs=-1)
+def subcommand(theory_id, datasets):
+    """Write EKO card for all FK tables in all datasets."""
+    paths = configs.configs["paths"]
+    for ds in datasets:
+        rich.print(f"Analyze {ds}")
+        grids = parser.load_grids(theory_id, ds)
+        for name, grid in grids.items():
+            opcard_path = paths["operator_cards"] / f"{name}.yaml"
+            _x_grid, q2_grid = evolve.write_operator_card_from_file(
+                grid, paths["opcard_template"], opcard_path
+            )
+            rich.print(
+                f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}"
+            )
+        print()

src/pineko/comparator.py

@@ -1,6 +1,6 @@
+# -*- coding: utf-8 -*-
 import numpy as np
 import pandas as pd
-
 import pineappl
 
 

src/pineko/configs.py

@@ -1,60 +1,17 @@
 # -*- coding: utf-8 -*-
+import copy
 import pathlib
-import typing
+import sys
+
 import appdirs
 import tomli
 
-import rich
-
 name = "pineko.toml"
 "Name of the config while (wherever it is placed)"
 
 
-class Configurations:
-    def __init__(self, dictionary=None):
-        if isinstance(dictionary, Configurations):
-            self._dict = dictionary._dict
-        elif dictionary is None:
-            self._dict = {}
-        else:
-            self._dict = dictionary
-
-    def __repr__(self):
-        return self._dict.__repr__()
-
-    def __getattribute__(self, name) -> typing.Any:
-        if name[0] == "_":
-            return super().__getattribute__(name)
-
-        value = self._dict[name]
-        if isinstance(value, dict):
-            value = Configurations(value)
-        return value
-
-    def __getitem__(self, key):
-        return self.__getattribute__(key)
-
-    def __setattribute__(self, name, value):
-        self._dict[name] = value
-
-    def __setitem__(self, key, value):
-        if key[0] == "_":
-            raise LookupError(
-                "Elements with leading '_' can not be retrieved later, so you"
-                f" can not set (attempted: '{key}')"
-            )
-
-        self._dict[key] = value
-
-    def __contains__(self, item):
-        return item in self._dict
-
-    def _pprint(self):
-        rich.print(self._dict)
-
-
 # better to declare immediately the correct type
-configs = Configurations()
+configs = {}
 "Holds loaded configurations"
 
 
@@ -75,24 +32,49 @@ def defaults(base_configs):
     ----
     The general rule is to never replace user provided input.
     """
-    configs = Configurations(base_configs)
+    configs = copy.deepcopy(base_configs)
 
     configs = add_paths(configs)
 
-    return Configurations(configs)
+    return configs
 
 
 def add_paths(configs):
-    for key, default in dict(grids="grids", ekos="ekos").items():
-        if key not in configs.paths:
-            configs.paths[key] = configs.paths.root / default
-        elif pathlib.Path(configs.paths[key]).anchor == "":
-            configs.paths[key] = configs.paths.root / configs.paths[key]
+    for key in [
+        "ymldb",
+        "operator_cards",
+        "grids",
+        "grids_common",
+        "opcard_template",
+        "theory_cards",
+        "fktables",
+        "ekos",
+    ]:
+        if key not in configs["paths"]:
+            raise ValueError(f"Configuration is missing a 'paths.{key}' key")
+        if pathlib.Path(configs["paths"][key]).anchor == "":
+            configs["paths"][key] = configs["paths"]["root"] / configs["paths"][key]
+        else:
+            configs["paths"][key] = pathlib.Path(configs["paths"][key])
+
+    if "logs" not in configs["paths"]:
+        configs["paths"]["logs"] = {}
+
+    for key in [
+        "eko",
+    ]:
+        if key not in configs["paths"]["logs"]:
+            configs["paths"]["logs"][key] = None
+        elif pathlib.Path(configs["paths"]["logs"][key]).anchor == "":
+            configs["paths"]["logs"][key] = (
+                configs["paths"]["root"] / configs["paths"]["logs"][key]
+            )
         else:
-            configs.paths[key] = pathlib.Path(configs.paths[key])
+            configs["paths"]["logs"][key] = pathlib.Path(configs["paths"]["logs"][key])
 
     return configs
 
+
 def detect(path=None):
     paths = []
 
@@ -122,7 +104,7 @@ def load(path=None):
             return {"paths": {"root": pathlib.Path.cwd()}}
         else:
             print("Configuration path specified is not valid.")
-            quit()
+            sys.exit()
 
     with open(path, "rb") as fd:
         loaded = tomli.load(fd)