Merge pull request #567 from wireapp/release/2019-01-10

Release/2019 01 10

CHANGELOG.md

@@ -1,3 +1,30 @@
+# 2019-01-10 #567
+
+## API changes
+
+  * `sigkeys` attribute on POST|PUT to `/clients` is now deprecated and ignored (clients can stop sending it)
+  * `cancel_callback` parameter on GET `/notifications` is now deprecated and ignored (clients can stop sending it)
+  * deprecated `POST /push/fallback/<notif>/cancel` is removed
+  * deprecated `tokenFallback` field returned on `GET /push/tokens` is removed
+
+## Bug fixes
+
+  * Size-restrict SSO subject identities (#557)
+  * Propagate team deletions to spar (#519)
+  * allow using $arg_name in nginz (#538)
+
+## Internal changes
+
+  * Version upgrades to GHC 8.4 (LTS-12), nginx 14.2, alpine 3.8 (#527, #540)
+  * Code refactoring, consitency with Imports.hs (#543, #553, #552)
+  * improved test coverage on spar (#539)
+  * Use yaml configuration in cannon (#555)
+
+## Others
+
+  * Docs and local dev/demo improvements
+
+
 # 2018-12-07 #542
 
 ## API changes

Makefile

@@ -1,7 +1,5 @@
 SHELL            := /usr/bin/env bash
 LANG             := en_US.UTF-8
-HASKELL_SERVICES := proxy cannon cargohold brig galley gundeck
-SERVICES         := $(HASKELL_SERVICES) nginz
 DOCKER_USER      ?= wireserver
 DOCKER_TAG       ?= local
 
@@ -89,11 +87,6 @@ i-%:
 #################################
 ## docker targets
 
-.PHONY: docker-services
-docker-services:
-	$(MAKE) -C build/alpine
-	$(foreach service,$(SERVICES),$(MAKE) -C services/$(service) docker;)
-
 .PHONY: docker-deps
 docker-deps:
 	$(MAKE) -C build/alpine deps

README.md

@@ -112,7 +112,9 @@ If you wish to build your own docker images, you need [docker version >= 17.05](
 make docker-services
 ```
 
-will, eventually, have built a range of docker images. See the `Makefile`s and `Dockerfile`s, as well as [build/alpine/README.md](build/alpine/README.md) for details.
+will, eventually, have built a range of docker images. Make sure to [give Docker enough RAM](https://github.com/wireapp/wire-server/issues/562); if you see `make: *** [builder] Error 137`, it might be a sign that the build ran out of memory. You can also mix and match – e.g. pull the [`alpine-builder`](https://quay.io/repository/wire/alpine-builder?tab=tags) image and build the rest locally.
+
+See the `Makefile`s and `Dockerfile`s, as well as [build/alpine/README.md](build/alpine/README.md) for details.
 
 ### How to run integration tests
 
@@ -143,13 +145,22 @@ make integration
 
 Or, alternatively, `make` on the top-level directory (to produce all the service's binaries) followed by e.g `cd services/brig && make integration` to run one service's integration tests only.
 
-You can use `$WIRE_STACK_OPTIONS` to pass arguments to stack through the `Makefile`s.  This is useful to e.g. pass arguments to tasty or temporarily disable `-Werror` without the risk of accidentally committing anything, like this:
+### when you need more fine-grained control over your build-test loops
+
+You can use `$WIRE_STACK_OPTIONS` to pass arguments to stack through the `Makefile`s.  This is useful to e.g. pass arguments to a unit test suite or temporarily disable `-Werror` without the risk of accidentally committing anything, like this:
+
+```bash
+WIRE_STACK_OPTIONS='--ghc-options=-Wwarn --test-arguments="--quickcheck-tests=19919 --quickcheck-replay=651712"' make -C services/gundeck
+```
+
+Integration tests are run via `/services/integration.sh`, which does not know about stack or `$WIRE_STACK_OPTIONS`.  Here you can use `$WIRE_INTEGRATION_TEST_OPTIONS`:
 
 ```bash
-WIRE_STACK_OPTIONS='--ghc-options=-Wwarn --test-arguments="--quickcheck-tests=19919 --quickcheck-replay=651712"' make integration
+cd services/spar
+WIRE_INTEGRATION_TEST_OPTIONS="--match='POST /identity-providers'" make i
 ```
 
-Note that [tasty supports passing arguments vie shell variables directly](https://github.com/feuerbach/tasty#runtime).
+Alternatively, you can use [tasty's support for passing arguments vie shell variables directly](https://github.com/feuerbach/tasty#runtime).  Or, in the case of spar, the [hspec equivalent](https://hspec.github.io/options.html#specifying-options-through-an-environment-variable), which [is less helpful at times](https://github.com/hspec/hspec/issues/335).
 
 ## How to install and run `wire-server`
 

build/alpine/Dockerfile.builder

@@ -5,14 +5,15 @@ ARG prebuilder=quay.io/wire/alpine-prebuilder
 FROM ${prebuilder}
 WORKDIR /src/wire-server
 
-# Get newer Stack
-
 # Download stack indices and compile/cache dependencies to speed up subsequent
 # container creation.
 #
 # We also build profiling versions of all libraries. Due to a bug in Stack,
 # they have to be built in a separate directory. See this issue:
 # https://github.com/commercialhaskell/stack/issues/4032
+#
+# Finally, we build docs for haskell-src-exts without hyperlinking enabled
+# to avoid a Haddock segfault. See https://github.com/haskell/haddock/issues/928
 
 RUN apk add --no-cache git ncurses && \
     mkdir -p /src && cd /src && \
@@ -29,5 +30,9 @@ RUN apk add --no-cache git ncurses && \
     echo '# use case, we want the docker container to write to its own'     >> /root/.stack/config.yaml && \
     echo '# stack-work directory and not pollute the one on the host.'      >> /root/.stack/config.yaml && \
     echo 'work-dir: .stack-docker'                                          >> /root/.stack/config.yaml && \
+    stack --work-dir .stack-docker-profile build --haddock --dependencies-only --profile haskell-src-exts && \
+    stack --work-dir .stack-docker         build --haddock --dependencies-only           haskell-src-exts && \
+    stack --work-dir .stack-docker-profile build --haddock --no-haddock-hyperlink-source --profile haskell-src-exts && \
+    stack --work-dir .stack-docker         build --haddock --no-haddock-hyperlink-source           haskell-src-exts && \
     stack --work-dir .stack-docker-profile build --pedantic --haddock --test --no-run-tests --bench --no-run-benchmarks --dependencies-only --profile && \
     stack --work-dir .stack-docker         build --pedantic --haddock --test --no-run-tests --bench --no-run-benchmarks --dependencies-only

build/alpine/Dockerfile.deps

@@ -1,6 +1,6 @@
 # Requires docker >= 17.05 (requires support for multi-stage builds)
 
-FROM alpine:3.7 as cryptobox-builder
+FROM alpine:3.8 as cryptobox-builder
 
 # compile cryptobox-c
 RUN apk add --no-cache cargo libsodium-dev git && \
@@ -10,7 +10,7 @@ RUN apk add --no-cache cargo libsodium-dev git && \
     cargo build --release
 
 # Minimal dependencies for alpine-compiled, dynamically linked wire-server Haskell services
-FROM alpine:3.7
+FROM alpine:3.8
 
 COPY --from=cryptobox-builder /tmp/cryptobox-c/target/release/libcryptobox.so /usr/lib
 

build/alpine/Dockerfile.prebuilder

@@ -1,6 +1,6 @@
 # Requires docker >= 17.05 (requires support for multi-stage builds)
 
-FROM alpine:3.7 as cryptobox-builder
+FROM alpine:3.8 as cryptobox-builder
 
 # compile cryptobox-c
 RUN apk add --no-cache cargo libsodium-dev git && \
@@ -9,7 +9,7 @@ RUN apk add --no-cache cargo libsodium-dev git && \
     cd cryptobox-c && \
     cargo build --release
 
-FROM alpine:3.7
+FROM alpine:3.8
 
 # install cryptobox-c in the new container
 COPY --from=cryptobox-builder /tmp/cryptobox-c/target/release/libcryptobox.so /usr/lib/libcryptobox.so
@@ -22,6 +22,8 @@ RUN apk add --no-cache \
         linux-headers \
         zlib-dev \
         ghc \
+        ghc-dev \
+        ghc-doc \
         libsodium-dev \
         openssl-dev \
         protobuf \
@@ -34,71 +36,7 @@ RUN apk add --no-cache \
         libxml2-dev
 
 # get static version of Haskell Stack and use system ghc by default
-ARG STACK_ALPINE_VERSION=1.6.3
+ARG STACK_ALPINE_VERSION=1.9.1
 RUN curl -sSfL https://github.com/commercialhaskell/stack/releases/download/v${STACK_ALPINE_VERSION}/stack-${STACK_ALPINE_VERSION}-linux-x86_64-static.tar.gz \
     | tar --wildcards -C /usr/local/bin --strip-components=1 -xzvf - '*/stack' && chmod 755 /usr/local/bin/stack && \
     stack config set system-ghc --global true
-
-# As done by https://github.com/TerrorJack/meikyu,
-# Install packages needed for newer version of GHC
-WORKDIR /root
-ENV LANG en_US.UTF-8
-ENV GHC_REV ghc-8.2.2-release
-ENV GHC_VER ghc-8.2.2
-ENV PATH /root/.local/bin:/root/.cabal/bin:/root/.stack/programs/x86_64-linux/$GHC_VER/bin:$PATH
-ADD ghc/build.mk ghc/config.yaml /tmp/
-RUN stack --no-terminal --resolver lts-9 --system-ghc install \
-        alex \
-        happy \
-        hscolour && \
-    apk add --no-cache --no-progress \
-    autoconf \
-    automake \
-    binutils-gold \
-    bzip2 \
-    ca-certificates \
-    coreutils \
-    file \
-    findutils \
-    g++ \
-    gawk \
-    gcc \
-    ghc \
-    git \
-    gmp-dev \
-    gzip \
-    libffi-dev \
-    make \
-    musl-dev \
-    ncurses-dev \
-    openssh \
-    patch \
-    perl \
-    py3-sphinx \
-    sed \
-    tar \
-    zlib-dev
-
-# Install newer version of GHC
-RUN cd /tmp && \
-    git clone git://git.haskell.org/ghc.git && \
-    cd ghc && \
-    git checkout $GHC_REV && \
-    git submodule update --init --recursive && \
-    mv /tmp/build.mk mk/
-
-RUN cd /tmp/ghc && \
-    ./boot && \
-    SPHINXBUILD=/usr/bin/sphinx-build-3 ./configure --prefix=/root/.stack/programs/x86_64-linux/$GHC_VER --disable-ld-override && \
-    echo "compiling GHC, may take an hour. Log output sent to /dev/null due to travis log length restrictions." && \
-    make -j4 &> /dev/null && \
-    make install &> /dev/null && \
-    mv /tmp/config.yaml /root/.stack/ && \
-    rm -rf /tmp/ghc
-
-# Upgrade stack (there is no binary for stack-1.7.1 on alpine, so this is the easiest way.)
-ARG STACK_VERSION=1.7.1
-RUN stack update && \
-    stack upgrade --binary-version ${STACK_VERSION} && \
-    rm -f /usr/local/bin/stack && \
-    cp /root/.local/bin/stack /usr/local/bin/stack

build/alpine/README.md

@@ -26,4 +26,6 @@ cd services/brig && make docker
 * `Dockerfile.intermediate` - based on `Dockerfile.deps`/`Dockerfile.builder`, this is an intermediate image compiling all dynamically linked binaries (obtained when running `make install` in the top-level directory).
 * `Dockerfile.executable` - based on `Dockerfile.deps`/`Dockerfile.intermediate`, this extracts a single executable from the intermediate image, yielding a small image with a single dynamically linked binary.
 * `Dockerfile.migrations` - same as `Dockerfile.executable`, with a fixed set of database migration binaries.
-* `Dockerfile.prebuilder` - dependencies of `Dockerfile.builder` that are expected to change very rarely (GHC, system libraries).
+* `Dockerfile.prebuilder` - dependencies of `Dockerfile.builder` that are expected to change very rarely (GHC, system libraries). Currently we're able to use system GHC, but if we require a newer version of GHC than the one provided by Alpine, we could build GHC in `Dockerfile.prebuilder` (as it has been [done before][2018-11-28]).
+
+[2018-11-28]: https://github.com/wireapp/wire-server/releases/tag/v2018-11-28

deploy/services-demo/conf/brig.demo.yaml

@@ -100,4 +100,4 @@ optSettings:
   setMaxConvSize: 128
 
 logLevel: Debug
-logNetStrings: true
+logNetStrings: false

deploy/services-demo/conf/cannon.demo.yaml

@@ -6,3 +6,6 @@ cannon:
 gundeck:
   host: 127.0.0.1
   port: 8086
+
+logLevel: Info
+logNetStrings: false

deploy/services-demo/conf/galley.demo.yaml

@@ -16,6 +16,10 @@ gundeck:
   host: 127.0.0.1
   port: 8086
 
+spar:
+  host: 127.0.0.1
+  port: 8088
+
 settings:
   httpPoolSize: 1024
   maxTeamSize: 128
@@ -24,4 +28,4 @@ settings:
   conversationCodeURI: https://127.0.0.1/join/
 
 logLevel: Info
-logNetStrings: true
+logNetStrings: false

deploy/services-demo/conf/gundeck.demo.yaml

@@ -20,13 +20,6 @@ aws:
   sqsEndpoint: http://localhost:4568 # https://sqs.eu-west-1.amazonaws.com
   snsEndpoint: http://localhost:4575 # https://sns.eu-west-1.amazonaws.com
 
-fallback:
-  skipFallbacks: true
-  preferNotice: true
-  queueDelay: 300
-  queueLimit: 30000
-  queueBurst: 100
-
 settings:
   httpPoolSize: 1024
   notificationTTL: 24192200

deploy/services-demo/conf/spar.demo.yaml

@@ -5,7 +5,7 @@ saml:
   spHost:    127.0.0.1
   spPort:    8088
   spAppUri:  http://localhost:8080/ # <--- change this to point to a reachable web app
-  spSsoUri:  http://localhost:8080/
+  spSsoUri:  http://localhost:8080/sso # <--- change this to the URL by which spar can be reached from an external IdP
 
   contacts:
     - type: ContactBilling
@@ -31,8 +31,6 @@ cassandra:
 maxttlAuthreq: 28800 # 8h
 maxttlAuthresp: 28800 # 8h
 
-logNetStrings: False # log using netstrings encoding (see http://cr.yp.to/proto/netstrings.txt)
+maxScimTokens: 16
 
-spInfo:
-  metaURI:  http://localhost:8088/sso/metadata
-  loginURI: http://localhost:8088/sso/initiate-login/
+logNetStrings: False # log using netstrings encoding (see http://cr.yp.to/proto/netstrings.txt)

deploy/services-demo/demo.sh

@@ -83,7 +83,7 @@ function run_haskell_service() {
     service=$1
     colour=$2
     # TODO can be removed once all services have been switched to YAML configs
-    export LOG_LEVEL=$3
+    [ $# -gt 2 ] && export LOG_LEVEL=$3
     (cd ${SCRIPT_DIR} && ${DIR}/../dist/${service} -c ${SCRIPT_DIR}/conf/${service}.demo.yaml || kill_all) \
         | sed -e "s/^/$(tput setaf ${colour})[${service}] /" -e "s/$/$(tput sgr0)/" &
 }
@@ -107,10 +107,10 @@ check_prerequisites
 run_haskell_service brig ${green}
 run_haskell_service galley ${yellow}
 run_haskell_service gundeck ${blue} Info
-run_haskell_service cannon ${orange} Info
+run_haskell_service cannon ${orange}
 run_haskell_service cargohold ${purpleish} Info
 run_haskell_service proxy ${redish} Info
-run_haskell_service spar ${orange} Info
+run_haskell_service spar ${orange}
 run_nginz ${blueish}
 
 sleep 3 # wait a moment for services to start before continuing

libs/api-bot/api-bot.cabal

@@ -67,7 +67,6 @@ library
       , monad-control         >= 1.0
       , mtl                   >= 2.1
       , mwc-random            >= 0.13
-      , old-locale            >= 1.0
       , optparse-applicative  >= 0.11
       , resource-pool         >= 0.2
       , scientific            >= 0.3

libs/api-bot/src/Network/Wire/Bot/Crypto/Glue.hs

@@ -5,7 +5,6 @@ module Network.Wire.Bot.Crypto.Glue
     , deleteBox
     , genPrekeys
     , genLastKey
-    , genSigKeys
     , randomBytes
     , unwrap
     ) where
@@ -46,12 +45,6 @@ genPrekey :: Box -> Word16 -> IO C.Prekey
 genPrekey box i = C.Prekey (C.PrekeyId i) . decodeUtf8 . encode <$>
     (CBox.copyBytes . CBox.prekey =<< unwrap =<< CBox.newPrekey box i)
 
-genSigKeys :: Box -> IO C.SignalingKeys
-genSigKeys box = do
-    let action = randomBytes box 32
-    C.SignalingKeys <$> (C.EncKey <$> action)
-                    <*> (C.MacKey <$> action)
-
 randomBytes :: MonadIO m => Box -> Word32 -> m ByteString
 randomBytes b n = liftIO $ CBox.randomBytes b n >>= unwrap >>= CBox.copyBytes
 

libs/api-bot/src/Network/Wire/Bot/Monad.hs

@@ -307,7 +307,6 @@ data BotClient = BotClient
     { botClientId       :: !ClientId
     , botClientLabel    :: !(Maybe Text)
     , botClientBox      :: !Box
-    , botClientSigKeys  :: !SignalingKeys
     , botClientSessions :: !Clients -- TODO: Map UserId (Map ClientId Session)
     }
 
@@ -328,20 +327,18 @@ addBotClient self cty label = do
     box <- liftIO $ openBox (userId $ botUser self) label
     pks <- liftIO $ genPrekeys box 100
     lk  <- liftIO $ genLastKey box
-    sg  <- liftIO $ genSigKeys box
     let nc = NewClient
            { newClientPassword = Just (botPassphrase self)
            , newClientPrekeys  = pks
            , newClientLastKey  = lk
-           , newClientSigKeys  = sg
            , newClientLabel    = label
            , newClientType     = cty
            , newClientClass    = Nothing
            , newClientCookie   = Nothing
            , newClientModel    = Nothing
            }
     cid <- clientId <$> runBotSession self (registerClient nc)
-    clt <- BotClient cid label box sg <$> liftIO Clients.empty
+    clt <- BotClient cid label box <$> liftIO Clients.empty
     liftIO . atomically $ modifyTVar' (botClients self) (clt:)
     return clt