Some helper docker scripts for easy local distributed hadoop testing

dockers/Dockerfile

@@ -0,0 +1,43 @@
+# Grabs the hadoop-VERSION-SNAPSHOT.tar.gz from hadoop-dist/target folder and construct a a HDFS docker
+# Assume that this script is ran from the dockers folder
+FROM eclipse-temurin:11-alpine
+
+# Arguments
+ARG VERSION=3.5.0
+ARG HADOOP_NAME=hadoop-$VERSION-SNAPSHOT
+
+# Copy over the tar.gz
+WORKDIR /opt
+COPY ./hadoop-dist/target/$HADOOP_NAME.tar.gz /opt
+
+# Unpack the build artifact
+RUN tar -xvf $HADOOP_NAME.tar.gz -C /opt
+RUN chmod +x /opt/$HADOOP_NAME/bin/hdfs
+RUN mkdir /opt/$HADOOP_NAME/logs
+COPY ./dockers/namenode-entrypoint.sh /opt/$HADOOP_NAME/sbin/
+RUN chmod +x /opt/$HADOOP_NAME/sbin/namenode-entrypoint.sh
+
+# Create a data directory for HDFS, we will point to this directory in config
+RUN mkdir -p /data/dataNode
+RUN mkdir -p /data/nameNode
+
+# bash is the preferred shell
+RUN apk update
+RUN apk add bash openssh sudo
+
+# Configure key based ssh
+RUN mkdir ~/.ssh
+COPY ./dockers/hadoop-id-rsa /root/.ssh
+COPY ./dockers/hadoop-id-rsa.pub /root/.ssh
+COPY ./dockers/hadoop-id-rsa.pub /root/.ssh/authorized_keys
+
+# Configure ENV variables
+ENV HADOOP_HOME=/opt/$HADOOP_NAME
+ENV PATH="$HADOOP_HOME/sbin:$HADOOP_HOME/bin:$PATH"
+ENV HDFS_DATANODE_USER=root
+ENV HDFS_NAMENODE_USER=root
+ENV HDFS_SECONDARYNAMENODE_USER=root
+
+ENV JAVA_HOME=/opt/java/openjdk
+
+

dockers/core-site.xml

@@ -0,0 +1,6 @@
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://namenode:9000</value>
+    </property>
+</configuration>

dockers/docker-compose.yml

@@ -0,0 +1,40 @@
+# This grabs hadoop-VERSION-snapshot.tar from the hadoop-dist/target directory and create a hadoop docker image
+version: '3.3'
+name: hadoop-four-nodes
+services:
+  namenode:
+    image: hdfs-runtime:latest
+    restart: always
+    environment:
+      HDFS_DATANODE_USER: root
+    ports:
+      - "9870:9870"
+    volumes:
+      - ./core-site.xml:/opt/hadoop-3.5.0-SNAPSHOT/etc/hadoop/core-site.xml
+      - ./hdfs-site.xml:/opt/hadoop-3.5.0-SNAPSHOT/etc/hadoop/hdfs-site.xml
+    entrypoint: bash namenode-entrypoint.sh
+    networks:
+      - intra
+
+  datanode:
+    image: hdfs-runtime:latest
+    restart: always
+    environment:
+      HDFS_DATANODE_USER: root
+    volumes:
+      - ./core-site.xml:/opt/hadoop-3.5.0-SNAPSHOT/etc/hadoop/core-site.xml
+      - ./hdfs-site.xml:/opt/hadoop-3.5.0-SNAPSHOT/etc/hadoop/hdfs-site.xml
+    entrypoint: hdfs datanode
+    networks:
+      - intra
+    deploy:
+      mode: replicated
+      replicas: 3
+
+networks:
+    intra:
+
+volumes:
+  test_volume_1:
+  test_volume_2:
+  test_volume_3:

dockers/hadoop-id-rsa

@@ -0,0 +1,38 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEA21jLtr+TCNuyrjcpPURaQCM6pcpNJKRkmS3YcZRpdibS55A3KtUr
+crSc2NBjXqoMm+N7CUT2T25qLH91eSQnUjTBi2muPXXBf7NfhRTjSXszfi1OxdDNYyy31v
++TkF8DHQBBahdhCCkqbbnUIddTt4p85yuEIxH3BKnMdfw0KbBdvMB4q3pRxbvWfKuWGmar
+Sj7dVWDlIR8LePPIYLtpkKlHA5LNOtygXCt7Z63TijHUG7c8ALgJ8RIKWP8dky3fu10yeo
+6rK1d8ZAiQpPXtgn4eCPIrLXyp73bVuXLC3Eux2uyVRZYDGB98s/xpcHvzS70XhTzeXznW
+rsy1LnDBith5lzTJowQXS5kjkvx5ABA/wbP61DpuFQdcqPeH1bj0ZTsr1iC9DfUvhiSVzC
+8YsJGbLxJTw/pJCl7vDwOGtzjwYMAbTXc3nZtQRPknnIjW3sRSNT719kELg/RaVWNWAXCS
+/0NRB4DwPxIWJJhfjVpvQgKOzFozUbjQ2ouC6aZ5AAAFkP4rzGb+K8xmAAAAB3NzaC1yc2
+EAAAGBANtYy7a/kwjbsq43KT1EWkAjOqXKTSSkZJkt2HGUaXYm0ueQNyrVK3K0nNjQY16q
+DJvjewlE9k9uaix/dXkkJ1I0wYtprj11wX+zX4UU40l7M34tTsXQzWMst9b/k5BfAx0AQW
+oXYQgpKm251CHXU7eKfOcrhCMR9wSpzHX8NCmwXbzAeKt6UcW71nyrlhpmq0o+3VVg5SEf
+C3jzyGC7aZCpRwOSzTrcoFwre2et04ox1Bu3PAC4CfESClj/HZMt37tdMnqOqytXfGQIkK
+T17YJ+HgjyKy18qe921blywtxLsdrslUWWAxgffLP8aXB780u9F4U83l851q7MtS5wwYrY
+eZc0yaMEF0uZI5L8eQAQP8Gz+tQ6bhUHXKj3h9W49GU7K9YgvQ31L4YklcwvGLCRmy8SU8
+P6SQpe7w8Dhrc48GDAG013N52bUET5J5yI1t7EUjU+9fZBC4P0WlVjVgFwkv9DUQeA8D8S
+FiSYX41ab0ICjsxaM1G40NqLgummeQAAAAMBAAEAAAGALZvCCAa/kPj1p0nq9kVPKsTofe
+6ZqcPQ8xTYS42bfCjIc8PF5M8cUKaDcoWqE9Xx9zwOqje/q3Z+9y+tgnnk/y/fCpj31Om2
+SVJHy0kzlWTbqSHMnk8zVarHsxnDWiKXaCFUeLXRxDujCdu44yx9yM0v+XVP7Wbf4QVZcT
+L2ubBlXUj8DlPJyv6qs5OOC4FLno2Bwi36lq942DaAdLSjV6QLExAkQlf7tA6Kg11eff6s
+ZvLrtA6QaqLgqe6p/J9t4lxlfkmLWCnWW74Sl+yKrpnC5HwMoaInvgt0jEza2tJ5ZE0Lyw
+ydptt1DNs7/Af96IpULjkXyh1WgOxWJ9DoPf2rwOIf0IqfiLD0uYGMdqmsvyYHKOM/4wmM
+MvUi0dxsTUjSExTgYYapRM7kbn93+DxJ02h49nCva/gCmHjajOW4P5z63gQvDle41fHBOY
+CxLWbx06ERM0kziJqKtEcnCMksgSzMS4OUH/m46EClFM1HTk/J24bAmq5kS0EZHGoxAAAA
+wQCYYcOIfYb+fs9Wz0vuu0YK/b5d7bowPK84SI9VW0P/ImL9f62oJa4FzdGaprSCev0eSH
+0asrWHtab/kNL3X5aEW0BNDE52g2bzp5xjeC8009nj0c+D8VDh40zOy1PnVpfJmRcienPL
+3KuIH4fRJ1bdToeU1yZF+Kv8kwogGew7y58Al/ouqhqUk0A9JknMAQQyBsewf001jGvFqL
+pdlVPBjo9MD2XUnZRASrzRMzJyeCk7bzoi2Urft42fCbfclkwAAADBAPGH8mX1iBuzU0uE
+tsJWcgH3MJmJNbW4RPvVVVuewPyghrr2pfszGtMnV+Z/9QumWU6MB5+RNB4dpY+mr9AqGW
+j5LboazP7PIqtReD4ADCGTE9ubyR/OY2bhsp2k88mn0p0Hst/nSV3RhBvbZgdWXFCEzi0v
+tGVjBcAYh2b1DyR6WyAITCd+pFuezk8Wwc5Mve731L4T/h685JzOSRKKFhdZ60BV2WV0hG
+e/L2/aXF58oyrDQjQ0YS/X/BGFInKshQAAAMEA6HyjbZGvjM99ldlEC9sxRmTldxg7c9hI
+KtvSURCSO7I6YiyGl9WVhqTY61A3UJogHNxcNxqPWTqwaYuFgZyaSe0XGf1v/sReRiJoic
+n8HDL6GY3iVaNHCPhsK/Ktw/rF9d47jSw3u84G6gKtZ2fgzqiTOgNNueYJP0bmyupSazHC
+OIXdyGoi+zkIlsIx5Pe4tjxIXWUm/RmUv+vv4aHv564RHHo0d2MqAPT9s2q1P9em8HiABm
+4lNzgwCWER5B5lAAAAFGFhcm9ubWFvQEppYWp1bi00MDkwAQIDBAUG
+-----END OPENSSH PRIVATE KEY-----

dockers/hadoop-id-rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbWMu2v5MI27KuNyk9RFpAIzqlyk0kpGSZLdhxlGl2JtLnkDcq1StytJzY0GNeqgyb43sJRPZPbmosf3V5JCdSNMGLaa49dcF/s1+FFONJezN+LU7F0M1jLLfW/5OQXwMdAEFqF2EIKSptudQh11O3inznK4QjEfcEqcx1/DQpsF28wHirelHFu9Z8q5YaZqtKPt1VYOUhHwt488hgu2mQqUcDks063KBcK3tnrdOKMdQbtzwAuAnxEgpY/x2TLd+7XTJ6jqsrV3xkCJCk9e2Cfh4I8istfKnvdtW5csLcS7Ha7JVFlgMYH3yz/Glwe/NLvReFPN5fOdauzLUucMGK2HmXNMmjBBdLmSOS/HkAED/Bs/rUOm4VB1yo94fVuPRlOyvWIL0N9S+GJJXMLxiwkZsvElPD+kkKXu8PA4a3OPBgwBtNdzedm1BE+SeciNbexFI1PvX2QQuD9FpVY1YBcJL/Q1EHgPA/EhYkmF+NWm9CAo7MWjNRuNDai4Lppnk= aaronmao@Jiajun-4090

dockers/hdfs-site.xml

@@ -0,0 +1,16 @@
+<configuration>
+    <property>
+            <name>dfs.namenode.name.dir</name>
+            <value>/data/nameNode</value>
+    </property>
+
+    <property>
+            <name>dfs.datanode.data.dir</name>
+            <value>/data/dataNode</value>
+    </property>
+
+    <property>
+            <name>dfs.replication</name>
+            <value>3</value>
+    </property>
+</configuration>

dockers/namenode-entrypoint.sh

@@ -0,0 +1,2 @@
+hdfs namenode -format -force
+hdfs namenode