Skip to content

Commit

Permalink
Add support for Singularity builds
Browse files Browse the repository at this point in the history 
Signed-off-by: Paolo Di Tommaso <[email protected]>
  • Loading branch information
@pditommaso
pditommaso committed Aug 19, 2023
1 parent e6d4bcc commit 6c4bedf
Show file tree
Hide file tree
Showing 7 changed files with 176 additions and 17 deletions.
4 changes: 2 additions & 2 deletions app/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ repositories {
}

dependencies {
implementation 'io.seqera:wave-api:0.3.1'
implementation 'io.seqera:wave-utils:0.3.1'
implementation 'io.seqera:wave-api:0.4.0'
implementation 'io.seqera:wave-utils:0.5.0'
implementation 'info.picocli:picocli:4.6.1'
implementation 'com.squareup.moshi:moshi:1.14.0'
implementation 'com.squareup.moshi:moshi-adapters:1.14.0'
Expand Down
6 changes: 6 additions & 0 deletions app/conf/resource-config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@
{
"pattern":"\\Qtemplates/conda/dockerfile-conda-packages.txt\\E"
},
{
"pattern":"\\Qtemplates/conda/singularityfile-conda-file.txt\\E"
},
{
"pattern":"\\Qtemplates/conda/singularityfile-conda-packages.txt\\E"
},
{
"pattern":"\\Qtemplates/spack/dockerfile-spack-file.txt\\E"
}
Expand Down
31 changes: 26 additions & 5 deletions app/src/main/java/io/seqera/wavelit/App.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,11 @@
import io.seqera.wave.config.SpackOpts;
import io.seqera.wave.util.DockerHelper;
import io.seqera.wave.util.Packer;
import io.seqera.wavelit.exception.BadClientResponseException;
import io.seqera.wavelit.exception.IllegalCliArgumentException;
import io.seqera.wavelit.json.JsonHelper;
import io.seqera.wavelit.util.BuildInfo;
import io.seqera.wavelit.util.Checkers;
import io.seqera.wavelit.util.CliVersionProvider;
import io.seqera.wavelit.util.YamlHelper;
import org.slf4j.LoggerFactory;
Expand All @@ -54,6 +56,8 @@
import static picocli.CommandLine.Command;
import static picocli.CommandLine.Option;

import static io.seqera.wave.util.DockerHelper.*;

/**
* Wavelit main class
*/
Expand Down Expand Up @@ -152,6 +156,9 @@ public class App implements Runnable {
@Option(names = {"-o","--output"}, paramLabel = "json|yaml", description = "Output format. One of: json, yaml.")
private String outputFormat;

@Option(names = {"-s","--singularity"}, paramLabel = "false", description = "Enable Singularity build (experimental)")
private boolean singularity;

private BuildContext buildContext;

private ContainerConfig containerConfig;
Expand All @@ -178,7 +185,7 @@ else if( result.isVersionHelpRequested() ) {
app.run();
}
}
catch (IllegalCliArgumentException | CommandLine.ParameterException e) {
catch (IllegalCliArgumentException | CommandLine.ParameterException | BadClientResponseException e) {
System.err.println(e.getMessage());
System.exit(1);
}
Expand Down Expand Up @@ -299,6 +306,9 @@ protected void validateArgs() {
if( !isEmpty(contextDir) && isEmpty(containerFile) )
throw new IllegalCliArgumentException("Option --context requires the use of a container file");

if( singularity && !freeze )
throw new IllegalCliArgumentException("Singularity build requires enabling freeze mode");

if( !isEmpty(contextDir) ) {
// check that a container file has been provided
if( isEmpty(containerFile) )
Expand Down Expand Up @@ -332,6 +342,7 @@ protected SubmitContainerTokenRequest createRequest() {
.withTowerAccessToken(towerToken)
.withTowerWorkspaceId(towerWorkspaceId)
.withTowerEndpoint(towerEndpoint)
.withFormat( singularity ? "sif" : null )
.withFreezeMode(freeze);
}

Expand Down Expand Up @@ -473,10 +484,20 @@ protected String containerFileBase64() {
final CondaOpts opts = new CondaOpts()
.withMambaImage(condaBaseImage)
.withCommands(condaRunCommands);
final String result = condaPackages!=null
? DockerHelper.condaPackagesToDockerFile(condaPackages.stream().collect(Collectors.joining(" ")), Arrays.asList(condaChannels.split(",")), opts)
: DockerHelper.condaFileToDockerFile(opts);
return encodeStringBase64(result);
if( !Checkers.isEmpty(condaPackages) ) {
final String packages0 = condaPackages.stream().collect(Collectors.joining(" "));
final List<String> channels0 = Arrays.asList(condaChannels.split(","));
final String result = singularity
? condaPackagesToSingularityFile(packages0, channels0, opts)
: condaPackagesToDockerFile(packages0, channels0, opts);
return encodeStringBase64(result);
}
else {
final String result = singularity
? condaFileToSingularityFile(opts)
: condaFileToDockerFile(opts);
return encodeStringBase64(result);
}
}

if( !isEmpty(spackFile) || spackPackages!=null ) {
Expand Down
3 changes: 2 additions & 1 deletion app/src/main/java/io/seqera/wavelit/Client.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import io.seqera.wave.api.SubmitContainerTokenRequest;
import io.seqera.wave.api.SubmitContainerTokenResponse;
import io.seqera.wavelit.config.RetryOpts;
import io.seqera.wavelit.exception.BadClientResponseException;
import io.seqera.wavelit.json.JsonHelper;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
Expand Down Expand Up @@ -87,7 +88,7 @@ SubmitContainerTokenResponse submit(SubmitContainerTokenRequest request) {
return JsonHelper.fromJson(resp.body(), SubmitContainerTokenResponse.class);
else {
String msg = String.format("Wave invalid response: [%s] %s", resp.statusCode(), resp.body());
throw new IllegalStateException(msg);
throw new BadClientResponseException(msg);
}
}
catch (IOException e) {
Expand Down
28 changes: 28 additions & 0 deletions app/src/main/java/io/seqera/wavelit/exception/BadClientResponseException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* Copyright (c) 2023, Seqera Labs.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*
* This Source Code Form is "Incompatible With Secondary Licenses", as
* defined by the Mozilla Public License, v. 2.0.
*/

package io.seqera.wavelit.exception;

/**
* @author Paolo Di Tommaso <[email protected]>
*/
public class BadClientResponseException extends RuntimeException {

public BadClientResponseException(String message) {
super(message);
}

public BadClientResponseException(String message, Throwable cause) {
super(message, cause);
}


}
16 changes: 11 additions & 5 deletions app/src/main/resources/io/seqera/wavelit/usage-examples.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,23 @@ Examples:
# Build a container with Dockerfile
wavelit -f Dockerfile --context build-context-dir/

# Build a Conda multi-packages container
# Build a container based on Conda packages
wavelit --conda-package bamtools=2.5.2 --conda-package samtools=1.17

# Build a Conda based container using arm64 architecture
# Build a container based on Conda packages using arm64 architecture
wavelit --conda-package fastp --platform linux/arm64

# Build a Conda based container using a prefix.dev served lock file
# Build a container based on Conda lock file served via prefix.dev service
wavelit --conda-package https://prefix.dev//envs/pditommaso/rnaseq-nf/y8g8ax6a4srv/conda-lock.yml

# Build a Spack based container
# Build a container based on Spack packages
wavelit --spack-package cowsay

# Build a container getting persistent image name
# Build a container getting a persistent image name
wavelit -i alpine --freeze --build-repo docker.io/user/repo --tower-token <YOUR TOKEN>

# Build a Singularity container and push it to an OCI registry
wavelit -f Singularityfile --singularity --freeze --build-repo docker.io/user/repo

# Build a Singularity container based on Conda packages
wavelit --conda-package bamtools=2.5.2 --singularity --freeze --build-repo docker.io/user/repo
105 changes: 101 additions & 4 deletions app/src/test/groovy/io/seqera/wavelit/AppCondaOptsTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ class AppCondaOptsTest extends Specification {
thrown(IllegalCliArgumentException)
}

def 'should create container file from conda file' () {
def 'should create docker file from conda file' () {
given:
def folder = Files.createTempDirectory('test')
def condaFile = folder.resolve('conda.yml');
Expand All @@ -107,6 +107,7 @@ class AppCondaOptsTest extends Specification {
COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml
RUN micromamba install -y -n base -f /tmp/conda.yml \\
&& micromamba clean -a -y
USER root
'''.stripIndent()
and:
new String(req.condaFile.decodeBase64()) == 'MY CONDA FILE'
Expand All @@ -116,7 +117,7 @@ class AppCondaOptsTest extends Specification {
}


def 'should create container file from conda package' () {
def 'should create docker file from conda package' () {
given:
def app = new App()
String[] args = ["--conda-package", "foo"]
Expand All @@ -132,12 +133,13 @@ class AppCondaOptsTest extends Specification {
micromamba install -y -n base -c seqera -c bioconda -c conda-forge -c defaults \\
foo \\
&& micromamba clean -a -y
USER root
'''.stripIndent()
and:
req.condaFile == null
}

def 'should create container file from conda package and custom options' () {
def 'should create docker file from conda package and custom options' () {
given:
def app = new App()
String[] args = [
Expand All @@ -161,8 +163,103 @@ class AppCondaOptsTest extends Specification {
micromamba install -y -n base -c alpha -c beta \\
foo bar \\
&& micromamba clean -a -y
USER root
RUN one
RUN two
RUN two
'''.stripIndent()

and:
req.condaFile == null
}


def 'should create singularity file from conda file' () {
given:
def folder = Files.createTempDirectory('test')
def condaFile = folder.resolve('conda.yml');
condaFile.text = 'MY CONDA FILE'
and:
def app = new App()
String[] args = ['--singularity', "--conda-file", condaFile.toString()]

when:
new CommandLine(app).parseArgs(args)
and:
def req = app.createRequest()
then:
new String(req.containerFile.decodeBase64()) == '''\
BootStrap: docker
From: mambaorg/micromamba:1.4.9
%files
{{wave_context_dir}}/conda.yml /tmp/conda.yml
%post
micromamba install -y -n base -f /tmp/conda.yml \\
&& micromamba clean -a -y
%environment
export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH"
'''.stripIndent()
and:
new String(req.condaFile.decodeBase64()) == 'MY CONDA FILE'

cleanup:
folder?.deleteDir()
}


def 'should create singularity file from conda package' () {
given:
def app = new App()
String[] args = ['--singularity', "--conda-package", "foo"]

when:
new CommandLine(app).parseArgs(args)
and:
def req = app.createRequest()
then:
new String(req.containerFile.decodeBase64()) == '''\
BootStrap: docker
From: mambaorg/micromamba:1.4.9
%post
micromamba install -y -n base -c seqera -c bioconda -c conda-forge -c defaults \\
foo \\
&& micromamba clean -a -y
%environment
export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH"
'''.stripIndent()
and:
req.condaFile == null
}

def 'should create singularity file from conda package and custom options' () {
given:
def app = new App()
String[] args = [
'--singularity',
"--conda-package", "foo",
"--conda-package", "bar",
"--conda-base-image", "my/mamba:latest",
"--conda-channels", "alpha,beta",
"--conda-run-command", "RUN one",
"--conda-run-command", "RUN two",
]

when:
new CommandLine(app).parseArgs(args)
and:
def req = app.createRequest()
then:
new String(req.containerFile.decodeBase64()) == '''\
BootStrap: docker
From: my/mamba:latest
%post
micromamba install -y -n base -c alpha -c beta \\
foo bar \\
&& micromamba clean -a -y
%environment
export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH"
%post
RUN one
RUN two
'''.stripIndent()

and:
Expand Down

0 comments on commit 6c4bedf

Please sign in to comment.