Release 1.6.0

- Refactored and reduced size of all images (on-demand installation, removed not needed packages, improved ENVs/LABELs to reduce numbers of layers)
- Cleanup images
- Add new on-demand installation and provisioning of services (eg. postfix, ssh...)
- Improved python console handling and output, also fixes dependency detection
- Renamed alpine-3 to alpine (symlink to keep backward compatibility)
- Add webdevops/php-official and also to webdevops/php and other images
- Introduce docker-image-info for gathering information about current used image (family, distribution, version ...)
- Introduce docker-service enable and docker-service disable to enable and disable services (eg. postfix, ssh ...)
- Fix supervisord message about passwordless http server (unix socket)
- Fix and improve tests (eg. in ipv6 environments)
- Add go-replace to replace sed/awk stuff
- Add webdevops/liquisoap
- Add LOG_STDOUT and LOG_STDERR (redirect docker output, also possible to prevent output using /dev/null)
- Set nginx log level to warn in production and info in nginx-dev images
- Move ansible to webdevops/base

Makefile

@@ -57,7 +57,7 @@ graph-full:
 	python ./bin/console generate:graph --all --filename docker-image-full-layout.gv
 
 documentation:
-	docker run -t -i --rm -p 8080:8000 -v "$$(pwd)/documentation/docs/:/opt/docs" -e "VIRTUAL_HOST=documentation.docker" -e "VIRTUAL_PORT=8000" webdevops/sphinx sphinx-autobuild --poll -H 0.0.0.0 /opt/docs html
+	docker run -t -i --rm -p 8000 -v "$$(pwd)/documentation/docs/:/opt/docs" -e "VIRTUAL_HOST=documentation.docker" -e "VIRTUAL_PORT=8000" webdevops/sphinx sphinx-autobuild --poll -H 0.0.0.0 /opt/docs html
 
 webdevops/bootstrap:
 	python ./bin/console docker:build --threads=auto --whitelist=webdevops/bootstrap

baselayout/usr/local/bin/apt-add-repository

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+apt-install software-properties-common
+add-apt-repository $@
+apt-get purge -y -f software-properties-common

baselayout/usr/local/bin/apt-install

@@ -7,12 +7,18 @@ set -o errexit   ## set -e : exit the script if any statement returns a non-true
 
 export DEBIAN_FRONTEND=noninteractive
 
-# Update apt cache
-apt-get update
+if [[ -f "/tmp/.apt-update" ]]; then
+    echo "Detected prefetched 'apt-get update'"
+    # Install packages
+    apt-get install -y -f --no-install-recommends $*
+else
+    # Update apt cache
+    apt-get update
 
-# Install packages
-apt-get install -y -f --no-install-recommends $*
+    # Install packages
+    apt-get install -y -f --no-install-recommends $*
 
-# Clear files (reduce snapshot size)
-rm -rf /var/lib/apt/lists/*
-apt-get clean -y
+    # Clear files (reduce snapshot size)
+    rm -rf /var/lib/apt/lists/*
+    apt-get clean -y
+fi

baselayout/usr/local/bin/apt-update

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -o pipefail  # trace ERR through pipes
+set -o errtrace  # trace ERR through 'time command' and other functions
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+apt-get update
+touch /tmp/.apt-update

baselayout/usr/local/bin/apt-upgrade

@@ -7,12 +7,18 @@ set -o errexit   ## set -e : exit the script if any statement returns a non-true
 
 export DEBIAN_FRONTEND=noninteractive
 
-# Update apt cache
-apt-get update
+if [[ -f "/tmp/.apt-update" ]]; then
+    echo "Detected prefetched 'apt-get update'"
+    # Update packages
+    apt-get dist-upgrade -y -f
+else
+    # Update apt cache
+    apt-get update
 
-# Install packages
-apt-get dist-upgrade -y -f
+    # Update packages
+    apt-get dist-upgrade -y -f
 
-# Clear files (reduce snapshot size)
-rm -rf /var/lib/apt/lists/*
-apt-get clean -y
+    # Clear files (reduce snapshot size)
+    rm -rf /var/lib/apt/lists/*
+    apt-get clean -y
+fi

baselayout/usr/local/bin/docker-image-cleanup

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+LSB_FAMILY=$(docker-image-info family)
+
+case "$LSB_FAMILY" in
+    Debian)
+        rm -f /tmp/.apt-update
+        apt-get autoremove -y -f
+        apt-get clean -y
+        rm -rf /var/lib/apt/lists/*
+        ;;
+
+    RedHat)
+        yum autoremove --assumeyes
+        yum clean all
+        ;;
+
+    Alpine)
+        find /var/lib/apk/ -mindepth 1 -delete
+        ;;
+
+    Arch)
+        pacman -Sc
+        ;;
+
+    *)
+        echo "ERROR: Distribution $LSB_FAMILY not supported"
+        exit 1
+        ;;
+esac
+
+find /tmp/ /var/log/ -mindepth 1 -delete
+rm -rf /root/.cache

baselayout/usr/local/bin/docker-image-info

@@ -0,0 +1,87 @@
+#!/bin/sh
+
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+help() {
+    if [ -n "$1" ]; then
+        echo "$1"
+        echo ""
+    fi
+
+    echo "Usage: $0 <argument>"
+    echo ""
+    echo "  Application arguments:"
+    echo "         family    Get distribution family"
+    echo "           dist    Get distribution name"
+    echo "   dist-version    Get distribution version"
+    echo "   dist-release    Get distribution release"
+    echo "  dist-codename    Get distribution codename"
+    echo "            lsb    Get lsb informations (if available)"
+    echo "       lsb-desc    Get lsb description (if available)"
+    echo "      buildtime    Get buildtime of docker image"
+    echo ""
+
+    exit $2
+
+}
+
+if [ "$#" -ne 1 ]; then
+        help "[ERROR] Invalid argument" 1
+fi
+
+INFO_FILE=""
+
+case "$1" in
+    dist-family|distribution-family|family)
+        INFO_FILE=/opt/docker/etc/.registry/image_info_distribution_family
+        ;;
+
+    dist|distribution)
+        INFO_FILE=/opt/docker/etc/.registry/image_info_distribution
+        ;;
+
+    dist-version|distribution-version)
+        INFO_FILE=/opt/docker/etc/.registry/image_info_distribution_version
+        ;;
+
+    dist-release|distribution-release)
+        INFO_FILE=/opt/docker/etc/.registry/image_info_lsb_release
+        ;;
+
+    dist-codename|distribution-codename)
+        INFO_FILE=/opt/docker/etc/.registry/image_info_lsb_codename
+        ;;
+
+    lsb)
+        INFO_FILE=/opt/docker/etc/.registry/image_info_lsb
+        ;;
+
+    lsb-desc|lsb-description)
+        INFO_FILE=/opt/docker/etc/.registry/image_info_lsb_description
+        ;;
+
+    buildtime)
+        INFO_FILE=/opt/docker/etc/.registry/image_info_buildtime
+        ;;
+
+    help)
+        help "" 0
+        ;;
+
+    *)
+        help "[ERROR] Invalid argument" 1
+        ;;
+esac
+
+if [ -n "$INFO_FILE" ]; then
+    if [ -f "$INFO_FILE" ]; then
+        cat -- "$INFO_FILE"
+    else
+        echo "[ERROR] Infomation file $INFO_FILE not found!"
+        echo "        Please run generate-dockerimage-info on docker image creation!"
+        exit 2
+    fi
+else
+    help "" 1
+fi

baselayout/usr/local/bin/generate-dockerimage-info

@@ -0,0 +1,96 @@
+#!/bin/sh
+
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+LSB_FAMILY=""
+
+#############################
+# Distribution detection
+#############################
+
+if [ -x "/usr/bin/apt-get" ]; then
+    # Debian family
+    LSB_FAMILY="Debian"
+
+elif [ -x "/bin/yum" ]; then
+    # RedHat family
+    LSB_FAMILY="RedHat"
+
+elif [ -x "/sbin/apk" ]; then
+    # Alpine family
+    LSB_FAMILY="Alpine"
+
+elif [ -f "/etc/arch-release" ]; then
+    # Alpine family
+    LSB_FAMILY="Arch"
+
+else
+    # Unknown
+    echo "ERROR: Distribution detection failed"
+    exit 1
+fi
+
+#############################
+# Install
+#############################
+
+case "$LSB_FAMILY" in
+    Debian)
+        apt-install lsb-release
+        ;;
+
+    RedHat)
+        yum-install redhat-lsb-core
+        ;;
+esac
+
+#############################
+# Set distribution information
+#############################
+
+echo "Detected $LSB_FAMILY"
+
+mkdir -p /opt/docker/etc/.registry/
+echo "$LSB_FAMILY" > /opt/docker/etc/.registry/image_info_distribution_family
+echo "$LSB_FAMILY" > /opt/docker/etc/.registry/image_info_distribution
+date +%s >/opt/docker/etc/.registry/image_info_buildtime
+
+# Create all files
+touch /opt/docker/etc/.registry/image_info_distribution_version
+touch /opt/docker/etc/.registry/image_info_lsb
+touch /opt/docker/etc/.registry/image_info_lsb_description
+touch /opt/docker/etc/.registry/image_info_lsb_release
+touch /opt/docker/etc/.registry/image_info_lsb_codename
+
+# Collect distribution specific informations
+case "$LSB_FAMILY" in
+    Debian|RedHat)
+        lsb_release -i -s > /opt/docker/etc/.registry/image_info_distribution
+        lsb_release -r -s > /opt/docker/etc/.registry/image_info_distribution_version
+        lsb_release -a > /opt/docker/etc/.registry/image_info_lsb
+        lsb_release -d -s > /opt/docker/etc/.registry/image_info_lsb_description
+        lsb_release -r -s > /opt/docker/etc/.registry/image_info_lsb_release
+        lsb_release -c -s > /opt/docker/etc/.registry/image_info_lsb_codename
+        ;;
+
+    Alpine)
+        cat /etc/alpine-release > /opt/docker/etc/.registry/image_info_distribution_version
+        ;;
+esac
+
+
+#############################
+# Uninstall
+#############################
+
+case "$LSB_FAMILY" in
+    Debian)
+        apt-get purge -y -f lsb-release
+        ;;
+
+    RedHat)
+        yum erase --assumeyes redhat-lsb-core
+        yum autoremove --assumeyes
+        ;;
+esac

baselayout/usr/local/bin/generate-locales

@@ -1,25 +1,28 @@
 #!/bin/bash
 
-set -o pipefail  # trace ERR through pipes
-set -o errtrace  # trace ERR through 'time command' and other functions
+set -o pipefail  ## trace ERR through pipes
+set -o errtrace  ## trace ERR through 'time command' and other functions
 set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
 set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
 
+IMAGE_DISTRIBUTION_FAMILY=$(docker-image-info family)
+IMAGE_DISTRIBUTION=$(docker-image-info distribution)
+IMAGE_DISTRIBUTION_VERSION=$(docker-image-info distribution-version)
 
 #######################################
 ## Debian
 #######################################
 
-if [[ "$(lsb_release -i -s)" == "Debian" ]]; then
+if [[ "$IMAGE_DISTRIBUTION" == "Debian" ]]; then
     /usr/local/bin/apt-install locales-all
 fi
 
 #######################################
 ## Ubuntu
 #######################################
 
-if [[ "$(lsb_release -i -s)" == "Ubuntu" ]]; then
-    if [[ "$(lsb_release -r -s | cut -f 1 -d .)" -ge "16" ]]; then
+if [[ "$IMAGE_DISTRIBUTION" == "Ubuntu" ]]; then
+    if [[ "$(echo $IMAGE_DISTRIBUTION_VERSION| cut -f 1 -d .)" -ge "16" ]]; then
         # Ubuntu 16.04 or later
         /usr/local/bin/apt-install locales-all
     else
@@ -33,12 +36,12 @@ fi
 ## RedHat family
 #######################################
 
-function localedefdebug() {
-    echo $*
-    localedef "$@"
-}
+if [[ "$IMAGE_DISTRIBUTION_FAMILY" == "RedHat" ]]; then
 
-if [[ -f /etc/redhat-release ]]; then
+    function localedefdebug() {
+        echo $*
+        localedef "$@"
+    }
 
     # Failing locales
     ## && localedefdebug -c -i bo_CN -f UTF-8 bo_CN.utf8 \

bin/webdevops/DockerfileUtility.py

@@ -79,6 +79,8 @@ def parse_docker_info_from_path(path):
         image_name = (image_name_info['image'] if 'image' in image_name_info else '')
         image_tag = (image_name_info['tag'] if 'tag' in image_name_info else '')
 
+        image_is_duplicate = False
+
         # check if path is linked
         if os.path.islink(os.path.dirname(path)):
             linked_image_name_info = ([m.groupdict() for m in path_regex.finditer(os.path.realpath(path))])[0]
@@ -88,6 +90,7 @@ def parse_docker_info_from_path(path):
             linked_image_tag = (linked_image_name_info['tag'] if 'tag' in linked_image_name_info else '')
 
             image_from = image_prefix + linked_image_repository + '/' + linked_image_name + ':' + linked_image_tag
+            image_is_duplicate = True
         else:
             image_from = parse_dockerfile_from_statement(path)
 
@@ -97,7 +100,8 @@ def parse_docker_info_from_path(path):
             'tag': image_tag,
             'repository': image_prefix + image_repository,
             'imageName': image_name,
-            'from': image_from
+            'from': image_from,
+            'duplicate': image_is_duplicate
         }
         return imageInfo
 

bin/webdevops/command/BaseCommand.py

@@ -23,6 +23,7 @@
 import multiprocessing
 from cleo import Command
 from webdevops import Configuration
+from ..doit.DoitReporter import DoitReporter
 
 class BaseCommand(Command):
     configuration = False
@@ -86,6 +87,8 @@ def startup(self):
 
         if 'dryRun' in self.configuration and self.configuration.get('dryRun'):
             options.append('dry-run')
+            DoitReporter.simulation_mode = True
+
 
         print 'Executing %s (%s)' % (self.name, ', '.join(options))
         print ''