Add support for persisting data in volumes

airflow.Dockerfile

@@ -20,6 +20,7 @@ RUN sed -i '34 i sed -i "s/hadoop:/\$HADOOP_HOST:/g" /home/airflow/hadoop/etc/ha
 # Setting AIRFLOW__WEBSERVER__BASE_URL did not get applied for unknown reasons, update the config file instead
 RUN sed -i "34 i sed -i 's#base_url = http://localhost:8080#base_url = http://localhost:8080/airflow#' /home/airflow/airflow/airflow.cfg" /startup.sh
 RUN sed -i "34 i sed -i 's#endpoint_url = http://localhost:8080#endpoint_url = http://localhost:8080/airflow#' /home/airflow/airflow/airflow.cfg" /startup.sh
+RUN sed -i '11 i chown -R postgres /var/lib/postgresql/' /startup.sh
 
 COPY airflow/ /home/airflow/airflow/
 COPY spark/ /home/airflow/airflow/python/

docker-compose.yml

@@ -24,19 +24,25 @@ services:
 
   user-db:
     image: postgres:16-alpine
+    volumes:
+      - user-db-data:/var/lib/postgresql/data
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
 
   hadoop:
     build:
       dockerfile: hadoop.Dockerfile
+    volumes:
+      - hadoop-data:/home/hadoop/hadoopdata
     ports:
       - "8088:8088"
       - "9870:9870"
 
   airflow:
     build:
       dockerfile: airflow.Dockerfile
+    volumes:
+      - airflow-db-data:/var/lib/postgresql
     depends_on:
       user-db:
         condition: service_started
@@ -45,3 +51,11 @@ services:
     environment:
       - HADOOP_HOST=hadoop
       - SPARK_HOST=airflow    # Define the address under which spark is reachable externally
+
+volumes:
+  user-db-data:
+    external: false
+  hadoop-data:
+    external: false
+  airflow-db-data:
+    external: false

hadoop.Dockerfile

@@ -3,6 +3,7 @@ FROM marcelmittelstaedt/spark_base:latest
 LABEL org.opencontainers.image.source=https://github.com/hfxbse/dhbw-big-data
 
 RUN sed -i '44,50 s/^#//' /startup.sh
+RUN sed -i '11 i chown -R hadoop /home/hadoop/hadoopdata' /startup.sh
 RUN head -n -9 /startup.sh > temp.sh ; mv temp.sh /startup.sh
 RUN echo "echo executing hiveserver2; sudo -u hadoop -H sh -c /home/hadoop/hive/bin/hiveserver2" >> /startup.sh
 RUN chmod +x /startup.sh