Merge pull request #901 from geometalab/feature/update-info

Feature/update info

.dockerignore

@@ -5,3 +5,5 @@ rq-dashboard/
 
 .*
 !.git
+Dockerfile.*
+docker-compose*.yml

.travis.yml

@@ -1,7 +1,5 @@
 language: python
 python:
-  - "3.4"
-  - "3.5"
   - "3.6"
 
 cache: pip

Dockerfile.frontend

@@ -1,13 +1,17 @@
-# this gdal image comes with support for FileGDB and has python3.4 already installed
-FROM geometalab/gdal-debian:latest
+# This GDAL image comes with support for FileGDB and has Python 3.6 already installed.
+# Based on image _/ubuntu.
+
+FROM geometalab/gdal-docker:v3.0.0
 
 USER root
 
 ENV PYTHONUNBUFFERED=non-empty-string PYTHONIOENCODING=utf-8 LC_ALL=C.UTF-8 LANG=C.UTF-8
 ENV DJANGO_OSMAXX_CONVERSION_SERVICE_USERNAME=default_user DJANGO_OSMAXX_CONVERSION_SERVICE_PASSWORD=default_password
 ENV NUM_WORKERS=5 DATABASE_HOST=frontenddatabase DATABASE_PORT=5432 APP_PORT=8000 APP_HOST=0.0.0.0
 
-RUN apt-get clean && DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+RUN DEBIAN_FRONTEND=noninteractive apt-get clean \
+    && DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     git \
     libpq-dev
 

Dockerfile.mediator

@@ -1,5 +1,7 @@
-# this gdal image comes with support for FileGDB and has python3.4 already installed
-FROM geometalab/gdal-debian:latest
+# This GDAL image comes with support for FileGDB and has Python 3.6 already installed.
+# Based on official Ubuntu docker image.
+
+FROM geometalab/gdal-docker:v3.0.0
 
 USER root
 
@@ -9,8 +11,8 @@ ENV NUM_WORKERS=5 DATABASE_HOST=mediatordatabase DATABASE_PORT=5432 APP_PORT=890
 
 MAINTAINER HSR Geometalab <[email protected]>
 
-RUN apt-get clean && DEBIAN_FRONTEND=noninteractive apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y\
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
 \
     libgeos-dev \
     libgeos++-dev \

Dockerfile.nginx

@@ -1,4 +1,5 @@
 FROM nginx:alpine
+
 COPY ./docker_entrypoint/nginx/default.conf.template /etc/nginx/conf.d/default.conf.template
 CMD DOMAIN_NAMES=$(echo $VIRTUAL_HOST | sed 's/,/ /g') envsubst '$DOMAIN_NAMES' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf \
     && cat /etc/nginx/conf.d/default.conf \

Dockerfile.worker

@@ -1,143 +1,114 @@
-# this gdal image comes with support for FileGDB and has python3.4 already installed
-FROM geometalab/gdal-debian:latest
+# This GDAL image comes with support for FileGDB and has Python 3.6 already installed.
+# Based on official Ubuntu docker image.
+
+FROM geometalab/gdal-docker:v3.0.0
 
 USER root
 
 ENV PYTHONUNBUFFERED=non-empty-string PYTHONIOENCODING=utf-8 LC_ALL=C.UTF-8 LANG=C.UTF-8
 
 # make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
-RUN apt-get update && apt-get install -y apt-utils locales && rm -rf /var/lib/apt/lists/* \
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y apt-utils locales gpg \
+    && rm -rf /var/lib/apt/lists/* \
     && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
-    && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 ################## SETUP POSTGIS DATABASE with UTF8 support #############
 # explicitly set user/group IDs
 RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres
 
-# grab gosu for easy step-down from root
-RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4
-RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \
-    && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/1.2/gosu-$(dpkg --print-architecture)" \
-    && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/1.2/gosu-$(dpkg --print-architecture).asc" \
-    && gpg --verify /usr/local/bin/gosu.asc \
-    && rm /usr/local/bin/gosu.asc \
-    && chmod +x /usr/local/bin/gosu \
-    && apt-get purge -y --auto-remove ca-certificates wget \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
 RUN mkdir /docker-entrypoint-initdb.d
 
-RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
+RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \
+    apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
 
-ENV PG_MAJOR 10
-ENV POSTGIS_MAJOR 2.4
+ENV PG_MAJOR 11
+ENV POSTGIS_MAJOR 2.5
 
-RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list
-
-RUN apt-get update \
+RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list \
+    && DEBIAN_FRONTEND=noninteractive apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql-common \
     && sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y\
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         postgresql-${PG_MAJOR} \
         postgresql-contrib-${PG_MAJOR} \
         postgresql-${PG_MAJOR}-postgis-${POSTGIS_MAJOR} \
         postgresql-${PG_MAJOR}-postgis-scripts \
         postgresql-server-dev-${PG_MAJOR} \
         postgresql-contrib-${PG_MAJOR} \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /var/run/postgresql && chown -R 999:999 /var/run/postgresql
+    && DEBIAN_FRONTEND=noninteractive apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /var/run/postgresql && chown -R 999:999 /var/run/postgresql
 
 ENV PATH /usr/lib/postgresql/$PG_MAJOR/bin:$PATH
 ENV PGDATA /var/lib/postgresql/data
 
-RUN mkdir -p $PGDATA && chown -R 999:999 /var/lib/postgresql
-
-RUN pg_createcluster --locale=en_US.UTF-8 -d $PGDATA ${PG_MAJOR} main
+RUN mkdir -p $PGDATA && chown -R 999:999 /var/lib/postgresql \
+    && pg_createcluster --locale=en_US.UTF-8 -d $PGDATA ${PG_MAJOR} main
 
 ################## END SETUP POSTGIS DATABASE with UTF8 support #############
 
-RUN apt-get clean && apt-get update && \
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y\
 \
-    make \
-    cmake \
-    g++ \
-    git-core\
-    subversion\
-    build-essential\
-    libxml2-dev\
-    libgeos-dev \
-    libgeos++-dev\
-    libpq-dev\
-    libboost-dev\
-    libboost-system-dev\
-    libboost-filesystem-dev\
-    libboost-thread-dev\
-    libexpat1-dev \
-    zlib1g-dev \
-    libbz2-dev\
-    libproj-dev\
-    libtool\
-    automake \
-    libprotobuf-c0-dev\
-    protobuf-c-compiler\
-    lua5.2 \
-    liblua5.2-0 \
-    liblua5.2-dev \
-    liblua5.1-0 \
-    zip \
-    osmctools \
-    wget \
-    binutils \
-    libgeoip1 \
-\
-    libicu-dev \
-    debhelper \
-\
-    default-jre \
-    libkakasi2-dev \
-    pandoc \
-    curl \
-    && apt-get clean \
+    make cmake g++ libboost-dev libboost-system-dev \
+    libboost-filesystem-dev libexpat1-dev zlib1g-dev \
+    libbz2-dev libpq-dev lua5.2 liblua5.2-dev \
+    libproj-dev \
+    curl git wget \
+    libstdc++6 osmctools \
+    && DEBIAN_FRONTEND=noninteractive apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+ENV LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/lib:${LD_LIBRARY_PATH}
+RUN ldconfig
+
 WORKDIR /root/osm2pgsql
 
 # OSM2PGSQL
-ENV OSM2PGSQL_VERSION=0.92.0
+ENV OSM2PGSQL_VERSION=0.96.0 CXXFLAGS=-DACCEPT_USE_OF_DEPRECATED_PROJ_API_H=1
 RUN mkdir src &&\
     cd src &&\
     GIT_SSL_NO_VERIFY=true git clone https://github.com/openstreetmap/osm2pgsql.git &&\
     cd osm2pgsql &&\
     git checkout ${OSM2PGSQL_VERSION} &&\
-    mkdir build &&\
+    mkdir -p build &&\
     cd build &&\
-    cmake ..&&\
+    cmake .. &&\
     make &&\
     make install
 
+# correcter/more portable would be:
+#    cmake .. &&\
+#    echo 'cmake worked' &&\
+#    cmake --build . &&\
+#    echo 'also make worked' &&\
+#    cmake --build . --target install
+
 WORKDIR /var/data/garmin/additional_data/
 # Fetch required additional data for Garmin as documented http://www.mkgmap.org.uk/download/mkgmap.html
-RUN wget -O /var/data/garmin/additional_data/bounds.zip http://osm2.pleiades.uni-wuppertal.de/bounds/latest/bounds.zip
-RUN wget -O /var/data/garmin/additional_data/sea.zip http://osm2.pleiades.uni-wuppertal.de/sea/latest/sea.zip
+RUN wget -O /var/data/garmin/additional_data/bounds.zip http://osm.thkukuk.de/data/bounds-latest.zip \
+    && wget -O /var/data/garmin/additional_data/sea.zip http://osm.thkukuk.de/data/sea-latest.zip
 
 ENV CODE /code
 WORKDIR $CODE
 
 # Install dependencies
-ENV LIBUTF8PROCVERSION 2.0.2-1
-RUN wget -O libutf8proc-dev.deb http://ftp.ch.debian.org/debian/pool/main/u/utf8proc/libutf8proc-dev_${LIBUTF8PROCVERSION}_amd64.deb
-RUN wget -O libutf8proc1.deb http://ftp.ch.debian.org/debian/pool/main/u/utf8proc/libutf8proc2_${LIBUTF8PROCVERSION}_amd64.deb
-RUN dpkg --install libutf8proc1.deb libutf8proc-dev.deb
-RUN rm libutf8proc1.deb libutf8proc-dev.deb
-
-RUN git clone https://github.com/giggls/mapnik-german-l10n.git mapnik-german-l10n \
-    && cd mapnik-german-l10n && git checkout v2.2.6 \
-    && make && make install && make clean
+ENV LIBUTF8PROCVERSION 2.3.0-1
+RUN wget -O libutf8proc-dev.deb http://ftp.ch.debian.org/debian/pool/main/u/utf8proc/libutf8proc-dev_${LIBUTF8PROCVERSION}_amd64.deb \
+    && wget -O libutf8proc1.deb http://ftp.ch.debian.org/debian/pool/main/u/utf8proc/libutf8proc2_${LIBUTF8PROCVERSION}_amd64.deb \
+    && dpkg --install libutf8proc1.deb libutf8proc-dev.deb \
+    && rm libutf8proc1.deb libutf8proc-dev.deb
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y pandoc libkakasi2-dev libicu-dev \
+    && git clone https://github.com/giggls/mapnik-german-l10n.git mapnik-german-l10n \
+    && cd mapnik-german-l10n && git checkout v2.5.1 \
+    && make && make install && make clean \
+    && DEBIAN_FRONTEND=noninteractive apt-get purge -y pandoc \
+    && DEBIAN_FRONTEND=noninteractive apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 ENV HOME /home/py
 

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2017 HSR Hochschule für Technik Rapperswil
+Copyright (c) 2015-2019 HSR Hochschule für Technik Rapperswil
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -54,11 +54,13 @@ pip-sync-all: requirements-all.txt
 
 .PHONY: tests-quick
 tests-quick: up-redis up-pg
-	./runtests.py $(PYTEST_ARGS)
+	docker build -t worker:test -f Dockerfile.worker .
+	docker run --link pg_tests:postgres --link redis-local:redis -e DJANGO_DB_PORT=5432 -e DJANGO_DB_HOST=postgres -e DJANGO_SETTINGS_MODULE= -v "$$(pwd):/code" --rm worker:test bash -c 'cp -r /code /tmp/code && cd /tmp/code/ && ./runtests.py $(PYTEST_ARGS)'
 
 .PHONY: tests-all
 tests-all: up-redis up-pg up-pg_translit
-	./runtests.py $(PYTEST_ARGS) --runslow
+	docker build -t worker:test -f Dockerfile.worker .
+	docker run --link pg_translit:translit --link pg_tests:postgres --link redis-local:redis  -e DJANGO_DB_PORT=5432 -e DJANGO_DB_HOST=postgres -e PG_TRANSLIT_PORT=5432 -e PG_TRANSLIT_HOST=translit -e DJANGO_SETTINGS_MODULE= -v "$$(pwd):/code" --rm worker:test bash -c 'cp -r /code /tmp/code && cd /tmp/code/ && ./runtests.py $(PYTEST_ARGS) --runslow'
 
 .PHONY: tox
 tox: up-redis up-pg up-pg_translit

README.md

@@ -103,7 +103,7 @@ docker-compose up
 
 Unsure which version is running?
 
-Go to `<your_ip>:8888/version/`.
+Go to `<your_ip>:8889/version/`.
 
 where `<your_ip>` is your public IP.
 

activate_local_development

@@ -1,2 +1,2 @@
-alias docker-compose="PUBLIC_LOCALHOST_IP=$(ip route get 1 | awk '{print $NF;exit}') DEPLOY_VERSION=$(git describe --dirty) docker-compose -f docker-compose.yml -f docker-compose-dev.yml"
+alias docker-compose="PUBLIC_LOCALHOST_IP=$(ip route get 1 | awk '{print $(NF-2);exit}') DEPLOY_VERSION=$(git describe --dirty) docker-compose -f docker-compose.yml -f docker-compose-dev.yml"
 alias deactivate_local_development="unalias docker-compose;unalias deactivate_local_development"

conversion_service/config/settings/common.py

@@ -22,43 +22,43 @@
 
 # APP CONFIGURATION
 # ------------------------------------------------------------------------------
-DJANGO_APPS = (
+DJANGO_APPS = [
     # Default Django apps:
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.sites',
     'django.contrib.staticfiles',
     'django.contrib.gis',
-)
-THIRD_PARTY_APPS = (
+]
+THIRD_PARTY_APPS = [
     # async execution worker
     'django_rq',
     # rest API Framework
     'rest_framework',
     'rest_framework_gis',
     'pbf_file_size_estimation',
-)
+]
 # Apps specific for this project go here.
-LOCAL_APPS = (
+LOCAL_APPS = [
     'osmaxx.version',
     'osmaxx.clipping_area',
     'osmaxx.conversion',
-)
+]
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
 
 # MIDDLEWARE CONFIGURATION
 # ------------------------------------------------------------------------------
-MIDDLEWARE_CLASSES = (
+MIDDLEWARE = [
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
-)
+]
 
 # MIGRATIONS CONFIGURATION
 # ------------------------------------------------------------------------------

conversion_service/config/settings/local.py

@@ -12,8 +12,8 @@
 
 # django-debug-toolbar
 # ------------------------------------------------------------------------------
-MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
-INSTALLED_APPS += ('debug_toolbar', )
+MIDDLEWARE += ['debug_toolbar.middleware.DebugToolbarMiddleware', ]
+INSTALLED_APPS += ['debug_toolbar', ]
 
 INTERNAL_IPS = env.tuple('DJANGO_INTERNAL_IPS', default=('127.0.0.1',))
 
@@ -26,7 +26,7 @@
 
 # django-extensions
 # ------------------------------------------------------------------------------
-INSTALLED_APPS += ('django_extensions', )
+INSTALLED_APPS += ['django_extensions', ]
 
 # TESTING
 # ------------------------------------------------------------------------------