CLI

containers/README.md

@@ -0,0 +1,46 @@
+# Build container image
+
+Use the `Containerfile` file in this folder to build the `bench-<what>`
+container image.
+
+### Using an image
+
+This is the easy part: you just need to download the
+[image](https://github.com/N3PDF/external/pkgs/container/) and run with your
+favorite container tool.
+
+Here the explicit commands with `podman` [[1]](#docker)
+
+```sh
+podman pull ghcr.io/n3pdf/<image-name>:latest
+podman run ghcr.io/n3pdf/<image-name>
+```
+
+<a name="docker">[1]</a>: to use `docker` instead you can simply replace `podman -> docker`, they have compatible subcommands
+
+### Build an image
+
+_Use case_: if a new external installer is added, or some external code is
+updated, to receive the updates you need to regenerate the container (since they
+are pre-installed in the image itself)
+
+Update `Containerfile` (if needed [[2]](#container-update)), hen rerun:
+
+```sh
+./build.bash <folder>
+podman tag <image-id> ghcr.io/n3pdf/<folder>:<version>
+# login to GitHub registry with user credentials (not organization), see [3]
+echo ${PAT} | podman login ghcr.io -u <username> --password-stdin
+# finally publish
+podman push ghcr.io/n3pdf/<folder>:<version>
+# and publish the new latest (all layers already available, it's just an alias)
+podman push ghcr.io/n3pdf/<folder>:latest
+```
+
+Where `<folder>` is the name of one of the subfolders in this directory.
+
+<a name="container-update">[2]</a>: you need to update the container both to
+spell out new externals, but also to upgrade python version, or other build time
+dependencies (e.g. `gfortran`)
+<a name="github-registry-docs">[3]</a>: official [GitHub registry
+docs](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)

containers/bench-dis/Containerfile

@@ -0,0 +1,46 @@
+FROM docker.io/python:3.9
+
+# create a dedicated user
+RUN /bin/bash -c "apt update; \
+                  apt install -y wget patch autoconf libtool g++ gfortran make"
+
+RUN /bin/bash -c "useradd -m dis"
+
+USER dis
+ENV HOME=/home/dis
+WORKDIR $HOME
+
+RUN /bin/bash -c "pip install numpy"
+
+COPY --chown=dis packages/lhapdf $HOME/lhapdf
+COPY --chown=dis packages/apfel $HOME/apfel
+COPY --chown=dis packages/fonlldis $HOME/fonlldis
+COPY --chown=dis packages/qcdnum $HOME/qcdnum
+COPY --chown=dis packages/xspace-bench $HOME/xspace-bench
+# TODO: remove this dirty trick
+COPY --chown=dis packages/.backup/ $HOME/.backup
+
+ENV PREFIX=$HOME/.local
+ENV PATH=$PATH:$HOME/.local/bin
+
+RUN /bin/bash -c "cd lhapdf; sh install.sh"
+RUN /bin/bash -c "cd apfel; sh install.sh"
+RUN /bin/bash -c "cd fonlldis; sh install.sh"
+RUN /bin/bash -c "cd qcdnum; sh install.sh"
+RUN /bin/bash -c "cd xspace-bench; sh install.sh"
+
+# --------------------------------------------------------- #
+
+FROM docker.io/python:3.9-slim
+
+RUN /bin/bash -c "useradd -m dis"
+
+USER dis
+ENV HOME=/home/dis
+WORKDIR $HOME
+
+ENV LD_LIBRARY_PATH=$HOME/.local/lib
+
+COPY --from=0 /usr/lib/x86_64-linux-gnu/libquadmath* /usr/lib/x86_64-linux-gnu/
+COPY --from=0 /usr/lib/x86_64-linux-gnu/libgfortran* /usr/lib/x86_64-linux-gnu/
+COPY --from=0 /home/dis/.local/ $HOME/.local

containers/bench-evol/Containerfile

@@ -0,0 +1,41 @@
+FROM docker.io/python:3.9
+
+# create a dedicated user
+RUN /bin/bash -c "apt update; \
+                  apt install -y wget patch autoconf libtool g++ gfortran make"
+
+RUN /bin/bash -c "useradd -m evol"
+
+USER evol
+ENV HOME=/home/evol
+WORKDIR $HOME
+
+RUN /bin/bash -c "pip install numpy"
+
+COPY --chown=evol packages/lhapdf $HOME/lhapdf
+COPY --chown=evol packages/apfel $HOME/apfel
+COPY --chown=evol packages/qcdnum $HOME/qcdnum
+COPY --chown=evol packages/pegasus $HOME/pegasus
+# TODO: remove this dirty trick
+COPY --chown=evol packages/.backup/ $HOME/.backup
+
+ENV PREFIX=$HOME/.local
+ENV PATH=$PATH:$HOME/.local/bin
+
+RUN /bin/bash -c "cd lhapdf; sh install.sh"
+RUN /bin/bash -c "cd apfel; sh install.sh"
+RUN /bin/bash -c "cd qcdnum; sh install.sh"
+RUN /bin/bash -c "cd pegasus; sh install.sh"
+
+# --------------------------------------------------------- #
+
+FROM docker.io/python:3.9-slim
+
+RUN /bin/bash -c "apt update; \
+                  apt install -y git curl"
+
+ENV LD_LIBRARY_PATH=/usr/local/lib
+
+COPY --from=0 /usr/lib/x86_64-linux-gnu/libquadmath* /usr/lib/x86_64-linux-gnu/
+COPY --from=0 /usr/lib/x86_64-linux-gnu/libgfortran* /usr/lib/x86_64-linux-gnu/
+COPY --from=0 /home/evol/.local/ /usr/local

containers/build.bash

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+pushd ..
+podman build -t ghcr.io/n3pdf/$1 -f containers/$1/Containerfile .
+popd

packages/.backup/update.sh

@@ -0,0 +1,15 @@
+BACKUP_DIR=$(realpath $(dirname $0))
+ROOT=$(realpath $BACKUP_DIR/..)
+
+echo "Refetching all sources from: '$ROOT'"
+echo "  -> store them in '$BACKUP_DIR'"
+echo
+
+for ext in $ROOT/*/; do
+  if [ ! -f $ext/variables.sh ]; then
+    continue
+  fi
+
+  . $ext/variables.sh
+  sh $ext/download.sh $URL $BACKUP_DIR/$TARBALL
+done

packages/.gitignore

@@ -0,0 +1,2 @@
+*.tar.gz
+!.backup/*.tar.gz

packages/README.md

@@ -0,0 +1,44 @@
+# External
+
+This repo provides installer for the following software. 
+
+- [APFEL](https://github.com/scarrazza/apfel)
+- [LHAPDF](https://lhapdf.hepforge.org)
+- [QCDNUM](https://www.nikhef.nl/~h24/qcdnum/)
+- [FONLLdis](https://github.com/NNPDF/external)
+- [xspace-bench](https://github.com/NNPDF/external)
+- [pegasus](https://www.nikhef.nl/~avogt/pegasus.html)
+
+Installers share most of their structure, but since differences are unavoidable
+(and they are short and should not change frequently), it is far more flexible
+to have a separate independent copy of install script for each external.
+
+## Structure
+
+Each software is packaged providing installers in the subfolders of this repo
+(each one name after the program, but all lower case).
+
+The structure of each folder is the following:
+
+- `README.md` (optional): brief description and notes
+- `install.sh`: the actual installer, meant to be run by shell (`sh install.sh`)
+- `clean.sh`: a cleaner to remove files generated by the installer
+- `*.patch`: one or more patches (tiny corrections, or extensions, to the
+  original source)
+
+#### Clean
+
+If you clone this as a `git` repository and you run `install.sh` from within its
+own folder you should be able to clean also by running:
+
+```sh
+git clean -fd
+```
+
+If instead you download the script, or you run the script out of the repo, then
+running `sh clean.sh` in the same folder should clean up.
+
+## Disclaimer
+
+We are not the original authors of any of these software, all the rights are owned
+by them, and maintenance is up to them as well.

packages/_cli/.gitignore

@@ -0,0 +1,3 @@
+build/
+n3pkg/packages/*
+!n3pkg/packages/.gitkeep

packages/_cli/Makefile

@@ -0,0 +1,28 @@
+PACKAGES := $(shell find .. -maxdepth 1 -type d -regex '../[a-z].*' -exec basename {} \;)
+
+.PHONY: build clean cleanall generate
+
+build: generate
+	go build -o build/n3pkg n3pkg/main.go
+
+.SECONDEXPANSION:
+
+# depends on
+# - files in the corresponding folder ('../%' expanded first)
+# - toplevel
+# - non-hidden
+# - with one of the following extensions: .sh, .patch, .grid
+n3pkg/packages/%.go: $$(shell find ../% -maxdepth 1 -not -path '*/.*' \( -name *.sh -o -name *.patch -o -name *.grid \))
+	@echo "Generate embedding for '$*'"
+	@sh generate/embed.sh $@ $^
+	@echo
+
+generate: $(patsubst %, n3pkg/packages/%.go, ${PACKAGES})
+	@sh generate/collect.sh n3pkg/packages/packages
+	@echo "Embeddings generated\n"
+
+clean:
+	rm -rf build
+
+cleanall: clean
+	git clean -fdx n3pkg/packages/

packages/_cli/generate/collect.sh

@@ -0,0 +1,15 @@
+PACKAGES=$(cat $1 | sort --unique)
+DEST=$(dirname $1)/main.go
+
+echo "package packages\n" >$DEST
+
+for kind in scripts patches grids; do
+  Kind=$(echo $kind | cut -c1 | tr [:lower:] [:upper:])$(echo $kind | cut -c2-)
+  echo "var ${Kind} = map[string]map[string]string{" >>$DEST
+  for pkg in $PACKAGES; do
+    echo "    \"${pkg}\": ${pkg}_${kind}," >>$DEST
+  done
+  echo "}\n" >>$DEST
+done
+
+go fmt $DEST >/dev/null

packages/_cli/generate/embed.sh

@@ -0,0 +1,58 @@
+DEST=$1
+shift
+DEPS=$@
+PKG=$(basename $DEST | cut -d'.' -f1 | tr "-" "_")
+
+SCRIPTS=
+PATCHES=
+GRIDS=
+
+echo "  dest: $DEST"
+for dep in $DEPS; do
+  echo "  $(basename $dep)"
+  case $dep in
+  *.sh)
+    SCRIPTS="${SCRIPTS} ${dep}"
+    ;;
+  *.patch)
+    PATCHES="${PATCHES} ${dep}"
+    ;;
+  *.grid)
+    GRIDS="${GRIDS} ${dep}"
+    ;;
+  esac
+done
+
+mkdir n3pkg/packages/$PKG
+
+echo "package packages\n\n" >$DEST
+echo "import _ \"embed\"" >>$DEST
+
+for kind in SCRIPTS PATCHES GRIDS; do
+  eval "files=\$$kind"
+  echo "var (" >>$DEST
+
+  mapname="${PKG}_$(echo $kind | tr [:upper:] [:lower:])"
+  map="var $mapname = map[string]string{\n"
+
+  for file in $files; do
+    stem=$(basename $file | rev | cut -d'.' -f2- | rev | tr "-" "_")
+    slug=$(basename $file | tr "." "_" | tr "-" "_")
+    var="${PKG}_${slug}"
+    name=$(basename $file)
+
+    ln $file $(dirname $DEST)/$PKG/$name
+    echo "    //go:embed $PKG/$name" >>$DEST
+    echo "    $var string" >>$DEST
+    map="$map    \"$stem\": $var,\n"
+
+  done
+
+  echo ")\n" >>$DEST
+  map="$map}\n"
+  echo $map >>$DEST
+done
+
+go fmt $DEST >/dev/null
+
+echo $PKG >>"$(dirname $DEST)/packages"

packages/_cli/go.mod

@@ -0,0 +1,26 @@
+module github.com/N3PDF/external/_cli
+
+go 1.17
+
+require (
+	github.com/spf13/cobra v1.3.0
+	github.com/spf13/viper v1.10.1
+)
+
+require (
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)