From b2bd3ec5f26ee56fa9f765b8b3ea5c81331bcec2 Mon Sep 17 00:00:00 2001 From: Auguste Nausedaite Date: Mon, 12 Dec 2022 09:34:10 +0000 Subject: [PATCH 1/7] Add changes to Dockerfile to support ARM64 --- .build/azure-pipelines-job-build.yml | 14 ++++++++++++-- Dockerfile | 7 ++++--- environments/environment9.1.14.yml | 8 ++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 environments/environment9.1.14.yml diff --git a/.build/azure-pipelines-job-build.yml b/.build/azure-pipelines-job-build.yml index fb83e4e..d4a70f6 100644 --- a/.build/azure-pipelines-job-build.yml +++ b/.build/azure-pipelines-job-build.yml @@ -25,7 +25,17 @@ jobs: $(DBCVER)-$(Build.BuildId) $(DBCVER) arguments: --build-arg DBCVER=$(DBCVER) - + - task: Docker@2 + displayName: Build + inputs: + command: build + containerRegistry: 'DockerHub' + repository: datathirstltd/dbconnect + tags: | + $(DBCVER)-$(Build.BuildId)-arm64 + $(DBCVER)-arm64 + arguments: --build-arg DBCVER=$(DBCVER) --build-arg CONDA_ARCH=aarch64 --build-arg TINI_ARCH=tini-arm64 + - task: Docker@2 displayName: Push inputs: @@ -34,4 +44,4 @@ jobs: repository: datathirstltd/dbconnect tags: | $(DBCVER)-$(Build.BuildId) - $(DBCVER) + $(DBCVER)-$(Build.BuildId)-arm64 diff --git a/Dockerfile b/Dockerfile index f06d406..9ea34c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ ARG USERNAME=vscode ARG USER_UID=1000 ARG USER_GID=$USER_UID ARG DBCVER +ARG TINI_ARCH=tini +ARG CONDA_ARCH=x86_64 # Configure apt ENV DEBIAN_FRONTEND=noninteractive @@ -27,7 +29,7 @@ RUN apt-get update --fix-missing && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${CONDA_ARCH}.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ /opt/conda/bin/conda clean -tipsy && \ @@ -37,7 +39,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86 ENV TINI_VERSION v0.16.1 -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_ARCH} /usr/bin/tini RUN chmod +x /usr/bin/tini ENTRYPOINT ["/usr/bin/tini", "--" ] # CMD @@ -69,4 +71,3 @@ ENV DEBIAN_FRONTEND=dialog # Allow for a consistant java home location for settings - image is changing over time RUN if [ ! -d "/docker-java-home" ]; then ln -s "${JAVA_HOME}" /docker-java-home; fi - diff --git a/environments/environment9.1.14.yml b/environments/environment9.1.14.yml new file mode 100644 index 0000000..bd4caf3 --- /dev/null +++ b/environments/environment9.1.14.yml @@ -0,0 +1,8 @@ +name: dbconnect +channels: + - conda-forge +dependencies: + - python=3.8.12 + - pip: + - databricks-connect==9.1.14 + - six From cbde7ae04ec6863d6b1f880d9c548807db8610fa Mon Sep 17 00:00:00 2001 From: Auguste Nausedaite Date: Mon, 12 Dec 2022 10:21:29 +0000 Subject: [PATCH 2/7] Fix images to push --- .build/azure-pipelines-job-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.build/azure-pipelines-job-build.yml b/.build/azure-pipelines-job-build.yml index d4a70f6..11ce3b8 100644 --- a/.build/azure-pipelines-job-build.yml +++ b/.build/azure-pipelines-job-build.yml @@ -44,4 +44,6 @@ jobs: repository: datathirstltd/dbconnect tags: | $(DBCVER)-$(Build.BuildId) + $(DBCVER) $(DBCVER)-$(Build.BuildId)-arm64 + $(DBCVER)-arm64 From 5d3494bf381f9bea09413e922c98b3007c720663 Mon Sep 17 00:00:00 2001 From: Auguste Nausedaite Date: Mon, 12 Dec 2022 14:11:35 +0000 Subject: [PATCH 3/7] Change base image for ARM arch to be able to build properly --- .build/azure-pipelines-job-build.yml | 29 ++++++------ Dockerfile | 8 ++-- Dockerfile_ARM64 | 71 ++++++++++++++++++++++++++++ azure-pipelines.yml | 8 ++-- 4 files changed, 93 insertions(+), 23 deletions(-) create mode 100644 Dockerfile_ARM64 diff --git a/.build/azure-pipelines-job-build.yml b/.build/azure-pipelines-job-build.yml index 11ce3b8..f282e77 100644 --- a/.build/azure-pipelines-job-build.yml +++ b/.build/azure-pipelines-job-build.yml @@ -18,7 +18,7 @@ jobs: - task: Docker@2 displayName: Build inputs: - command: build + command: build for AMD architecture containerRegistry: 'DockerHub' repository: datathirstltd/dbconnect tags: | @@ -26,24 +26,25 @@ jobs: $(DBCVER) arguments: --build-arg DBCVER=$(DBCVER) - task: Docker@2 - displayName: Build + displayName: build for ARM architecture inputs: command: build + Dockerfile: Dockerfile_ARM64 containerRegistry: 'DockerHub' repository: datathirstltd/dbconnect tags: | $(DBCVER)-$(Build.BuildId)-arm64 $(DBCVER)-arm64 - arguments: --build-arg DBCVER=$(DBCVER) --build-arg CONDA_ARCH=aarch64 --build-arg TINI_ARCH=tini-arm64 + arguments: --build-arg DBCVER=$(DBCVER) - - task: Docker@2 - displayName: Push - inputs: - command: push - containerRegistry: 'DockerHub' - repository: datathirstltd/dbconnect - tags: | - $(DBCVER)-$(Build.BuildId) - $(DBCVER) - $(DBCVER)-$(Build.BuildId)-arm64 - $(DBCVER)-arm64 + # - task: Docker@2 + # displayName: Push + # inputs: + # command: push + # containerRegistry: 'DockerHub' + # repository: datathirstltd/dbconnect + # tags: | + # $(DBCVER)-$(Build.BuildId) + # $(DBCVER) + # $(DBCVER)-$(Build.BuildId)-arm64 + # $(DBCVER)-arm64 diff --git a/Dockerfile b/Dockerfile index 9ea34c6..cc9b670 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. #------------------------------------------------------------------------------------------------------------- -FROM openjdk:8-jdk +FROM arm64v8/openjdk:8-jdk # This Dockerfile adds a non-root user with sudo access. Use the "remoteUser" # property in devcontainer.json to use it. On Linux, the container user's GID/UIDs @@ -12,8 +12,6 @@ ARG USERNAME=vscode ARG USER_UID=1000 ARG USER_GID=$USER_UID ARG DBCVER -ARG TINI_ARCH=tini -ARG CONDA_ARCH=x86_64 # Configure apt ENV DEBIAN_FRONTEND=noninteractive @@ -29,7 +27,7 @@ RUN apt-get update --fix-missing && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${CONDA_ARCH}.sh -O ~/miniconda.sh && \ +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ /opt/conda/bin/conda clean -tipsy && \ @@ -39,7 +37,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${C ENV TINI_VERSION v0.16.1 -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_ARCH} /usr/bin/tini +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini RUN chmod +x /usr/bin/tini ENTRYPOINT ["/usr/bin/tini", "--" ] # CMD diff --git a/Dockerfile_ARM64 b/Dockerfile_ARM64 new file mode 100644 index 0000000..c29d712 --- /dev/null +++ b/Dockerfile_ARM64 @@ -0,0 +1,71 @@ +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- +FROM arm64v8/openjdk:8-jdk + +# This Dockerfile adds a non-root user with sudo access. Use the "remoteUser" +# property in devcontainer.json to use it. On Linux, the container user's GID/UIDs +# will be updated to match your local UID/GID (when using the dockerFile property). +# See https://aka.ms/vscode-remote/containers/non-root-user for details. +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=$USER_UID +ARG DBCVER + +# Configure apt +ENV DEBIAN_FRONTEND=noninteractive + +# Mini Conda Start +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV PATH /opt/conda/bin:$PATH +RUN echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc + + +RUN apt-get update --fix-missing && \ + apt-get install -y wget bzip2 ca-certificates curl git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + /opt/conda/bin/conda clean -tipsy && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate dbconnect" >> ~/.bashrc + + +ENV TINI_VERSION v0.16.1 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-arm64 /usr/bin/tini +RUN chmod +x /usr/bin/tini +ENTRYPOINT ["/usr/bin/tini", "--" ] +# CMD + + +COPY environments/environment${DBCVER}.yml . +RUN pip install --upgrade pip \ + && conda env create -f environment${DBCVER}.yml \ + && echo '{}' > /root/.databricks-connect + +# VSCode DevContainers +RUN apt-get update \ + && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 \ + # Create a non-root user to use if preferred - see https://aka.ms/vscode-remote/containers/non-root-user. + && groupadd --gid $USER_GID $USERNAME \ + && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \ + # [Optional] Add sudo support for the non-root user + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME\ + && chmod 0440 /etc/sudoers.d/$USERNAME \ + # Verify git, needed tools installed + && apt-get -y install git iproute2 procps curl lsb-release + +# Clean up +RUN apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* +ENV DEBIAN_FRONTEND=dialog + +# Allow for a consistant java home location for settings - image is changing over time +RUN if [ ! -d "/docker-java-home" ]; then ln -s "${JAVA_HOME}" /docker-java-home; fi diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 07f5c93..8af4330 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,10 +8,10 @@ stages: - stage: Build displayName: Build image jobs: - - template: .build/azure-pipelines-job-build.yml - parameters: - DBCVER: 9.1.13 - DBCVERNODOTS: 9113 + # - template: .build/azure-pipelines-job-build.yml + # parameters: + # DBCVER: 9.1.14 + # DBCVERNODOTS: 9114 - template: .build/azure-pipelines-job-build.yml parameters: From 3c267d348a87bbad3b8cd92c16b55f1dbac34556 Mon Sep 17 00:00:00 2001 From: Auguste Nausedaite Date: Mon, 12 Dec 2022 14:12:47 +0000 Subject: [PATCH 4/7] revert Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cc9b670..9354a17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. #------------------------------------------------------------------------------------------------------------- -FROM arm64v8/openjdk:8-jdk +FROM openjdk:8-jdk # This Dockerfile adds a non-root user with sudo access. Use the "remoteUser" # property in devcontainer.json to use it. On Linux, the container user's GID/UIDs From 10601dfc729e916b6e82d203cf8faa0cdee875bc Mon Sep 17 00:00:00 2001 From: Auguste Nausedaite Date: Mon, 12 Dec 2022 14:14:58 +0000 Subject: [PATCH 5/7] Fix azure pipeline typos --- .build/azure-pipelines-job-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.build/azure-pipelines-job-build.yml b/.build/azure-pipelines-job-build.yml index f282e77..5b30ee4 100644 --- a/.build/azure-pipelines-job-build.yml +++ b/.build/azure-pipelines-job-build.yml @@ -18,7 +18,7 @@ jobs: - task: Docker@2 displayName: Build inputs: - command: build for AMD architecture + command: Build for AMD architecture containerRegistry: 'DockerHub' repository: datathirstltd/dbconnect tags: | @@ -26,7 +26,7 @@ jobs: $(DBCVER) arguments: --build-arg DBCVER=$(DBCVER) - task: Docker@2 - displayName: build for ARM architecture + displayName: Build for ARM architecture inputs: command: build Dockerfile: Dockerfile_ARM64 From 9d460b54ee68c969ce2fe22060ac65879a94dff2 Mon Sep 17 00:00:00 2001 From: Auguste Nausedaite Date: Mon, 12 Dec 2022 14:17:24 +0000 Subject: [PATCH 6/7] Fix azure pipeline typos --- .build/azure-pipelines-job-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.build/azure-pipelines-job-build.yml b/.build/azure-pipelines-job-build.yml index 5b30ee4..043c040 100644 --- a/.build/azure-pipelines-job-build.yml +++ b/.build/azure-pipelines-job-build.yml @@ -16,9 +16,9 @@ jobs: command: login - task: Docker@2 - displayName: Build + displayName: Build for AMD architecture inputs: - command: Build for AMD architecture + command: build containerRegistry: 'DockerHub' repository: datathirstltd/dbconnect tags: | From 9f36f2594eca5e5ece7c6f33f0d623a3bf34e7b6 Mon Sep 17 00:00:00 2001 From: Auguste Nausedaite Date: Mon, 12 Dec 2022 14:29:08 +0000 Subject: [PATCH 7/7] Add platform arg for ARM --- .build/azure-pipelines-job-build.yml | 2 +- azure-pipelines.yml | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.build/azure-pipelines-job-build.yml b/.build/azure-pipelines-job-build.yml index 043c040..1bceb4a 100644 --- a/.build/azure-pipelines-job-build.yml +++ b/.build/azure-pipelines-job-build.yml @@ -35,7 +35,7 @@ jobs: tags: | $(DBCVER)-$(Build.BuildId)-arm64 $(DBCVER)-arm64 - arguments: --build-arg DBCVER=$(DBCVER) + arguments: --build-arg DBCVER=$(DBCVER) --platform linux/arm64/v8 # - task: Docker@2 # displayName: Push diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8af4330..ba63d51 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,10 +8,10 @@ stages: - stage: Build displayName: Build image jobs: - # - template: .build/azure-pipelines-job-build.yml - # parameters: - # DBCVER: 9.1.14 - # DBCVERNODOTS: 9114 + - template: .build/azure-pipelines-job-build.yml + parameters: + DBCVER: 9.1.14 + DBCVERNODOTS: 9114 - template: .build/azure-pipelines-job-build.yml parameters: @@ -22,8 +22,3 @@ stages: parameters: DBCVER: 10.4.6 DBCVERNODOTS: 1046 - - - template: .build/azure-pipelines-job-build.yml - parameters: - DBCVER: 7.3.38 - DBCVERNODOTS: 7338