-
Notifications
You must be signed in to change notification settings - Fork 194
/
Dockerfile
148 lines (117 loc) · 5.48 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
# Multi-container Dockerfile for build and run containers for vg
# Use Google's non-rate-limited mirror of Docker Hub to get our base image.
# This helps automated Quay builds because Quay hasn't built a caching system
# and exposes pull rate limits to users.
FROM mirror.gcr.io/library/ubuntu:20.04 AS base
MAINTAINER vgteam
RUN echo base > /stage.txt
WORKDIR /vg
# Prevent dpkg from trying to ask any questions, ever
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN true
FROM base AS build
ARG THREADS=8
ARG TARGETARCH
# If you didn't `make version` berfore building the Docker, you can provide a
# version value here to claim to be.
ARG VG_GIT_VERSION
ENV VG_GIT_VERSION=${VG_GIT_VERSION:-unknown}
RUN echo build > /stage.txt
RUN apt-get -qq -y update && \
apt-get -qq -y upgrade && \
apt-get -qq -y install sudo
# Install all vg's dependencies.
# The Makefile will come parse the Dockerfile to get the correct dependencies;
# this is the One True Depencency List.
# We don't need to clean the package index since we don't ship this image and
# don't care about its size.
# We clip out everything between these begin and end markers, except the line
# that starts with RUN, or comments. And we pull out line continuation slashes.
# TODO: can we read them here and in the Makefile from the README instead?
###DEPS_BEGIN###
RUN apt-get -qq -y update && apt-get -qq -y upgrade && apt-get -qq -y install \
make git build-essential protobuf-compiler libprotoc-dev libjansson-dev libbz2-dev \
libncurses5-dev automake gettext autopoint libtool jq bsdmainutils bc rs parallel npm \
samtools curl unzip redland-utils librdf-dev cmake pkg-config wget gtk-doc-tools \
raptor2-utils rasqal-utils bison flex gawk libgoogle-perftools-dev liblz4-dev liblzma-dev \
libcairo2-dev libpixman-1-dev libffi-dev libcairo-dev libprotobuf-dev libboost-all-dev \
tabix bcftools libzstd-dev pybind11-dev python3-pybind11
###DEPS_END###
# Prepare to build submodule dependencies
COPY deps /vg/deps
# To increase portability of the docker image, when building for amd64, set the
# target CPU architecture to Nehalem (2008) rather than auto-detecting the
# build machine's CPU. This has no AVX1, AVX2, or PCLMUL, but it does have
# SSE4.2. UCSC has a Nehalem machine that we want to support.
RUN if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then sed -i s/march=native/march=nehalem/ deps/sdsl-lite/CMakeLists.txt; fi
# Clear any CMake caches in case we are building from someone's checkout
RUN find . -name CMakeCache.txt | xargs rm -f
# Build the dependencies
COPY Makefile /vg/Makefile
RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" CFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) deps
# Bring in the sources, which we need in order to build.
COPY src /vg/src
# Build all the object files for vg, but don't link.
# Also pass the arch here
RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) objs
# Do the final build and link, knowing the version. Trim down the resulting binary but make sure to include enough debug info for profiling.
RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) static && strip -d bin/vg
# Ship the scripts
COPY scripts /vg/scripts
ENV PATH /vg/bin:$PATH
############################################################################################
FROM build AS test
ARG THREADS=8
RUN echo test > /stage.txt
RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qq -y install nodejs && npm install -g [email protected]
# Fail if any non-portable instructions were used
RUN /bin/bash -e -c 'if objdump -d /vg/bin/vg | grep vperm2i128 ; then exit 1 ; else exit 0 ; fi'
# Bring in the tests and docs, which have doctests
COPY test /vg/test
COPY doc /vg/doc
# We test the README so bring it along.
COPY README.md /vg/
# Run tests in the middle so the final container that gets tagged is the run container.
# Tests may not actually be run by smart builders like buildkit.
RUN /bin/bash -e -c "export OMP_NUM_THREADS=$((THREADS < $(nproc) ? THREADS : $(nproc))); make test"
############################################################################################
FROM base AS run
RUN echo run > /stage.txt
# Install packages which toil-vg needs to be available inside the image, for
# pipes and profiling, and good usability on Kubernetes.
# TODO: which of these can be removed?
# Make sure to clean so we don't ship old apt package indexes in our Docker.
RUN ls -lah /vg && \
apt-get -qq -y update && \
apt-get -qq -y upgrade && \
apt-get -qq -y install --no-upgrade \
curl \
wget \
pigz \
dstat \
pv \
jq \
samtools \
tabix \
parallel \
fontconfig-config \
awscli \
binutils \
libpython2.7 \
libperl-dev \
libelf1 \
libdw1 \
libslang2 \
libnuma1 \
numactl \
bc \
linux-tools-common \
linux-tools-generic \
perl \
time \
&& apt-get -qq -y clean
COPY --from=build /vg/bin/vg /vg/bin/
COPY --from=build /vg/scripts/* /vg/scripts/
# Make sure we have the flame graph scripts so we can do self-profiling
COPY --from=build /vg/deps/FlameGraph /vg/deps/FlameGraph
ENV PATH /vg/bin:$PATH