From 6c4bedf8ecd3eda815c4c51862ed4b59002ce8cc Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 19 Aug 2023 22:19:40 +0200 Subject: [PATCH] Add support for Singularity builds Signed-off-by: Paolo Di Tommaso --- app/build.gradle | 4 +- app/conf/resource-config.json | 6 + app/src/main/java/io/seqera/wavelit/App.java | 31 +++++- .../main/java/io/seqera/wavelit/Client.java | 3 +- .../exception/BadClientResponseException.java | 28 +++++ .../io/seqera/wavelit/usage-examples.txt | 16 ++- .../io/seqera/wavelit/AppCondaOptsTest.groovy | 105 +++++++++++++++++- 7 files changed, 176 insertions(+), 17 deletions(-) create mode 100644 app/src/main/java/io/seqera/wavelit/exception/BadClientResponseException.java diff --git a/app/build.gradle b/app/build.gradle index 65ed631..0db1fe5 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -20,8 +20,8 @@ repositories { } dependencies { - implementation 'io.seqera:wave-api:0.3.1' - implementation 'io.seqera:wave-utils:0.3.1' + implementation 'io.seqera:wave-api:0.4.0' + implementation 'io.seqera:wave-utils:0.5.0' implementation 'info.picocli:picocli:4.6.1' implementation 'com.squareup.moshi:moshi:1.14.0' implementation 'com.squareup.moshi:moshi-adapters:1.14.0' diff --git a/app/conf/resource-config.json b/app/conf/resource-config.json index 5325b76..806b20e 100644 --- a/app/conf/resource-config.json +++ b/app/conf/resource-config.json @@ -22,6 +22,12 @@ { "pattern":"\\Qtemplates/conda/dockerfile-conda-packages.txt\\E" }, + { + "pattern":"\\Qtemplates/conda/singularityfile-conda-file.txt\\E" + }, + { + "pattern":"\\Qtemplates/conda/singularityfile-conda-packages.txt\\E" + }, { "pattern":"\\Qtemplates/spack/dockerfile-spack-file.txt\\E" } diff --git a/app/src/main/java/io/seqera/wavelit/App.java b/app/src/main/java/io/seqera/wavelit/App.java index 3299b57..58c054f 100644 --- a/app/src/main/java/io/seqera/wavelit/App.java +++ b/app/src/main/java/io/seqera/wavelit/App.java @@ -40,9 +40,11 @@ import io.seqera.wave.config.SpackOpts; import io.seqera.wave.util.DockerHelper; import io.seqera.wave.util.Packer; +import io.seqera.wavelit.exception.BadClientResponseException; import io.seqera.wavelit.exception.IllegalCliArgumentException; import io.seqera.wavelit.json.JsonHelper; import io.seqera.wavelit.util.BuildInfo; +import io.seqera.wavelit.util.Checkers; import io.seqera.wavelit.util.CliVersionProvider; import io.seqera.wavelit.util.YamlHelper; import org.slf4j.LoggerFactory; @@ -54,6 +56,8 @@ import static picocli.CommandLine.Command; import static picocli.CommandLine.Option; +import static io.seqera.wave.util.DockerHelper.*; + /** * Wavelit main class */ @@ -152,6 +156,9 @@ public class App implements Runnable { @Option(names = {"-o","--output"}, paramLabel = "json|yaml", description = "Output format. One of: json, yaml.") private String outputFormat; + @Option(names = {"-s","--singularity"}, paramLabel = "false", description = "Enable Singularity build (experimental)") + private boolean singularity; + private BuildContext buildContext; private ContainerConfig containerConfig; @@ -178,7 +185,7 @@ else if( result.isVersionHelpRequested() ) { app.run(); } } - catch (IllegalCliArgumentException | CommandLine.ParameterException e) { + catch (IllegalCliArgumentException | CommandLine.ParameterException | BadClientResponseException e) { System.err.println(e.getMessage()); System.exit(1); } @@ -299,6 +306,9 @@ protected void validateArgs() { if( !isEmpty(contextDir) && isEmpty(containerFile) ) throw new IllegalCliArgumentException("Option --context requires the use of a container file"); + if( singularity && !freeze ) + throw new IllegalCliArgumentException("Singularity build requires enabling freeze mode"); + if( !isEmpty(contextDir) ) { // check that a container file has been provided if( isEmpty(containerFile) ) @@ -332,6 +342,7 @@ protected SubmitContainerTokenRequest createRequest() { .withTowerAccessToken(towerToken) .withTowerWorkspaceId(towerWorkspaceId) .withTowerEndpoint(towerEndpoint) + .withFormat( singularity ? "sif" : null ) .withFreezeMode(freeze); } @@ -473,10 +484,20 @@ protected String containerFileBase64() { final CondaOpts opts = new CondaOpts() .withMambaImage(condaBaseImage) .withCommands(condaRunCommands); - final String result = condaPackages!=null - ? DockerHelper.condaPackagesToDockerFile(condaPackages.stream().collect(Collectors.joining(" ")), Arrays.asList(condaChannels.split(",")), opts) - : DockerHelper.condaFileToDockerFile(opts); - return encodeStringBase64(result); + if( !Checkers.isEmpty(condaPackages) ) { + final String packages0 = condaPackages.stream().collect(Collectors.joining(" ")); + final List channels0 = Arrays.asList(condaChannels.split(",")); + final String result = singularity + ? condaPackagesToSingularityFile(packages0, channels0, opts) + : condaPackagesToDockerFile(packages0, channels0, opts); + return encodeStringBase64(result); + } + else { + final String result = singularity + ? condaFileToSingularityFile(opts) + : condaFileToDockerFile(opts); + return encodeStringBase64(result); + } } if( !isEmpty(spackFile) || spackPackages!=null ) { diff --git a/app/src/main/java/io/seqera/wavelit/Client.java b/app/src/main/java/io/seqera/wavelit/Client.java index 6814cb1..3c64edd 100644 --- a/app/src/main/java/io/seqera/wavelit/Client.java +++ b/app/src/main/java/io/seqera/wavelit/Client.java @@ -29,6 +29,7 @@ import io.seqera.wave.api.SubmitContainerTokenRequest; import io.seqera.wave.api.SubmitContainerTokenResponse; import io.seqera.wavelit.config.RetryOpts; +import io.seqera.wavelit.exception.BadClientResponseException; import io.seqera.wavelit.json.JsonHelper; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -87,7 +88,7 @@ SubmitContainerTokenResponse submit(SubmitContainerTokenRequest request) { return JsonHelper.fromJson(resp.body(), SubmitContainerTokenResponse.class); else { String msg = String.format("Wave invalid response: [%s] %s", resp.statusCode(), resp.body()); - throw new IllegalStateException(msg); + throw new BadClientResponseException(msg); } } catch (IOException e) { diff --git a/app/src/main/java/io/seqera/wavelit/exception/BadClientResponseException.java b/app/src/main/java/io/seqera/wavelit/exception/BadClientResponseException.java new file mode 100644 index 0000000..09ffeb5 --- /dev/null +++ b/app/src/main/java/io/seqera/wavelit/exception/BadClientResponseException.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023, Seqera Labs. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * This Source Code Form is "Incompatible With Secondary Licenses", as + * defined by the Mozilla Public License, v. 2.0. + */ + +package io.seqera.wavelit.exception; + +/** + * @author Paolo Di Tommaso + */ +public class BadClientResponseException extends RuntimeException { + + public BadClientResponseException(String message) { + super(message); + } + + public BadClientResponseException(String message, Throwable cause) { + super(message, cause); + } + + +} diff --git a/app/src/main/resources/io/seqera/wavelit/usage-examples.txt b/app/src/main/resources/io/seqera/wavelit/usage-examples.txt index dccc8bc..8856ace 100644 --- a/app/src/main/resources/io/seqera/wavelit/usage-examples.txt +++ b/app/src/main/resources/io/seqera/wavelit/usage-examples.txt @@ -6,17 +6,23 @@ Examples: # Build a container with Dockerfile wavelit -f Dockerfile --context build-context-dir/ - # Build a Conda multi-packages container + # Build a container based on Conda packages wavelit --conda-package bamtools=2.5.2 --conda-package samtools=1.17 - # Build a Conda based container using arm64 architecture + # Build a container based on Conda packages using arm64 architecture wavelit --conda-package fastp --platform linux/arm64 - # Build a Conda based container using a prefix.dev served lock file + # Build a container based on Conda lock file served via prefix.dev service wavelit --conda-package https://prefix.dev//envs/pditommaso/rnaseq-nf/y8g8ax6a4srv/conda-lock.yml - # Build a Spack based container + # Build a container based on Spack packages wavelit --spack-package cowsay - # Build a container getting persistent image name + # Build a container getting a persistent image name wavelit -i alpine --freeze --build-repo docker.io/user/repo --tower-token + + # Build a Singularity container and push it to an OCI registry + wavelit -f Singularityfile --singularity --freeze --build-repo docker.io/user/repo + + # Build a Singularity container based on Conda packages + wavelit --conda-package bamtools=2.5.2 --singularity --freeze --build-repo docker.io/user/repo diff --git a/app/src/test/groovy/io/seqera/wavelit/AppCondaOptsTest.groovy b/app/src/test/groovy/io/seqera/wavelit/AppCondaOptsTest.groovy index ecce35d..348de60 100644 --- a/app/src/test/groovy/io/seqera/wavelit/AppCondaOptsTest.groovy +++ b/app/src/test/groovy/io/seqera/wavelit/AppCondaOptsTest.groovy @@ -88,7 +88,7 @@ class AppCondaOptsTest extends Specification { thrown(IllegalCliArgumentException) } - def 'should create container file from conda file' () { + def 'should create docker file from conda file' () { given: def folder = Files.createTempDirectory('test') def condaFile = folder.resolve('conda.yml'); @@ -107,6 +107,7 @@ class AppCondaOptsTest extends Specification { COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml RUN micromamba install -y -n base -f /tmp/conda.yml \\ && micromamba clean -a -y + USER root '''.stripIndent() and: new String(req.condaFile.decodeBase64()) == 'MY CONDA FILE' @@ -116,7 +117,7 @@ class AppCondaOptsTest extends Specification { } - def 'should create container file from conda package' () { + def 'should create docker file from conda package' () { given: def app = new App() String[] args = ["--conda-package", "foo"] @@ -132,12 +133,13 @@ class AppCondaOptsTest extends Specification { micromamba install -y -n base -c seqera -c bioconda -c conda-forge -c defaults \\ foo \\ && micromamba clean -a -y + USER root '''.stripIndent() and: req.condaFile == null } - def 'should create container file from conda package and custom options' () { + def 'should create docker file from conda package and custom options' () { given: def app = new App() String[] args = [ @@ -161,8 +163,103 @@ class AppCondaOptsTest extends Specification { micromamba install -y -n base -c alpha -c beta \\ foo bar \\ && micromamba clean -a -y + USER root RUN one - RUN two + RUN two + '''.stripIndent() + + and: + req.condaFile == null + } + + + def 'should create singularity file from conda file' () { + given: + def folder = Files.createTempDirectory('test') + def condaFile = folder.resolve('conda.yml'); + condaFile.text = 'MY CONDA FILE' + and: + def app = new App() + String[] args = ['--singularity', "--conda-file", condaFile.toString()] + + when: + new CommandLine(app).parseArgs(args) + and: + def req = app.createRequest() + then: + new String(req.containerFile.decodeBase64()) == '''\ + BootStrap: docker + From: mambaorg/micromamba:1.4.9 + %files + {{wave_context_dir}}/conda.yml /tmp/conda.yml + %post + micromamba install -y -n base -f /tmp/conda.yml \\ + && micromamba clean -a -y + %environment + export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" + '''.stripIndent() + and: + new String(req.condaFile.decodeBase64()) == 'MY CONDA FILE' + + cleanup: + folder?.deleteDir() + } + + + def 'should create singularity file from conda package' () { + given: + def app = new App() + String[] args = ['--singularity', "--conda-package", "foo"] + + when: + new CommandLine(app).parseArgs(args) + and: + def req = app.createRequest() + then: + new String(req.containerFile.decodeBase64()) == '''\ + BootStrap: docker + From: mambaorg/micromamba:1.4.9 + %post + micromamba install -y -n base -c seqera -c bioconda -c conda-forge -c defaults \\ + foo \\ + && micromamba clean -a -y + %environment + export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" + '''.stripIndent() + and: + req.condaFile == null + } + + def 'should create singularity file from conda package and custom options' () { + given: + def app = new App() + String[] args = [ + '--singularity', + "--conda-package", "foo", + "--conda-package", "bar", + "--conda-base-image", "my/mamba:latest", + "--conda-channels", "alpha,beta", + "--conda-run-command", "RUN one", + "--conda-run-command", "RUN two", + ] + + when: + new CommandLine(app).parseArgs(args) + and: + def req = app.createRequest() + then: + new String(req.containerFile.decodeBase64()) == '''\ + BootStrap: docker + From: my/mamba:latest + %post + micromamba install -y -n base -c alpha -c beta \\ + foo bar \\ + && micromamba clean -a -y + %environment + export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" + %post + RUN one + RUN two '''.stripIndent() and: