add app installation into workspace

H3mul · Dec 6, 2023 · 67d81a3 · 67d81a3
1 parent cf91ea6
commit 67d81a3
Show file tree

Hide file tree

Showing 12 changed files with 65 additions and 141 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,11 +1,5 @@
 # Stage 1: Base
-FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04 as base
-
-ENV INSTALL_ROOT=/workspace/apps
-ENV WEBUI_VERSION=v1.6.0
-ENV DREAMBOOTH_COMMIT=cf086c536b141fc522ff11f6cffc8b7b12da04b9
-ENV KOHYA_VERSION=v22.2.1
-ENV INVOKEAI_VERSION=v3.4.0post2
+FROM python:3.10
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive \
@@ -23,10 +17,6 @@ RUN apt update && \
         rclone \
         build-essential \
         software-properties-common \
-        python3.10-venv \
-        python3-pip \
-        python3-tk \
-        python3-dev \
         nodejs \
         npm \
         parallel \
@@ -66,97 +56,6 @@ RUN apt update && \
     rm -rf /var/lib/apt/lists/* && \
     echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
 
-# Set Python
-RUN ln -s /usr/bin/python3.10 /usr/bin/python
-
-# # Clone the git repo of the Stable Diffusion Web UI by Automatic1111
-# # and set version
-# WORKDIR /
-# RUN git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui.git && \
-#     cd /stable-diffusion-webui && \
-#     git checkout tags/${WEBUI_VERSION}
-
-# WORKDIR /stable-diffusion-webui
-# COPY a1111/requirements.txt a1111/requirements_versions.txt a1111/install-automatic.py ./
-# RUN python3 -m venv --system-site-packages venv && \
-#     source venv/bin/activate && \
-#     pip3 install --no-cache-dir torch==2.0.1 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 && \
-#     pip3 install --no-cache-dir xformers==0.0.22 tensorrt && \
-#     pip3 install --no-cache-dir -r requirements_versions.txt && \
-#     python3 -m install-automatic --skip-torch-cuda-test && \
-#     deactivate
-
-# # Clone the Automatic1111 Extensions
-# RUN git clone https://github.com/d8ahazard/sd_dreambooth_extension.git extensions/sd_dreambooth_extension && \
-#     git clone --depth=1 https://github.com/deforum-art/sd-webui-deforum.git extensions/deforum && \
-#     git clone --depth=1 https://github.com/Mikubill/sd-webui-controlnet.git extensions/sd-webui-controlnet && \
-#     git clone --depth=1 https://github.com/ashleykleynhans/a1111-sd-webui-locon.git extensions/a1111-sd-webui-locon && \
-#     git clone --depth=1 https://github.com/Gourieff/sd-webui-reactor.git extensions/sd-webui-reactor && \
-#     git clone --depth=1 https://github.com/zanllp/sd-webui-infinite-image-browsing.git extensions/infinite-image-browsing && \
-#     git clone --depth=1 https://github.com/Uminosachi/sd-webui-inpaint-anything.git extensions/inpaint-anything && \
-#     git clone --depth=1 https://github.com/Bing-su/adetailer.git extensions/adetailer
-#     # git clone --depth=1 https://github.com/civitai/sd_civitai_extension.git extensions/sd_civitai_extension && \
-#     # git clone --depth=1 https://github.com/BlafKing/sd-civitai-browser-plus.git extensions/sd-civitai-browser-plus
-
-# # Install dependencies for Deforum, ControlNet, ReActor, Infinite Image Browsing,
-# # After Detailer, and CivitAI Browser+ extensions
-# RUN source venv/bin/activate && \
-#     cd /stable-diffusion-webui/extensions/deforum && \
-#     pip3 install -r requirements.txt && \
-#     cd /stable-diffusion-webui/extensions/sd-webui-controlnet && \
-#     pip3 install -r requirements.txt && \
-#     cd /stable-diffusion-webui/extensions/sd-webui-reactor && \
-#     pip3 install -r requirements.txt && \
-#     pip3 install onnxruntime-gpu && \
-#     cd /stable-diffusion-webui/extensions/infinite-image-browsing && \
-#     pip3 install -r requirements.txt && \
-#     cd /stable-diffusion-webui/extensions/adetailer && \
-#     python3 -m install && \
-#     # cd /stable-diffusion-webui/extensions/sd_civitai_extension && \
-#     # pip3 install -r requirements.txt && \
-#     deactivate
-
-# # Install dependencies for inpaint anything extension
-# RUN source venv/bin/activate && \
-#     pip3 install segment_anything lama_cleaner && \
-#     deactivate
-
-# # Install dependencies for Civitai Browser+ extension
-# # RUN source venv/bin/activate && \
-# #     cd /stable-diffusion-webui/extensions/sd-civitai-browser-plus && \
-# #     pip3 install send2trash ZipUnicode && \
-# #     deactivate
-
-# # Set Dreambooth extension version
-# WORKDIR /stable-diffusion-webui/extensions/sd_dreambooth_extension
-# RUN git checkout main && \
-#     git reset ${DREAMBOOTH_COMMIT} --hard
-
-# # Install the dependencies for the Dreambooth extension
-# WORKDIR /stable-diffusion-webui
-# COPY a1111/requirements_dreambooth.txt ./requirements.txt
-# RUN source venv/bin/activate && \
-#     cd /stable-diffusion-webui/extensions/sd_dreambooth_extension && \
-#     pip3 install -r requirements.txt && \
-#     deactivate
-
-# # Add inswapper model for the ReActor extension
-# RUN mkdir -p /stable-diffusion-webui/models/insightface && \
-#     cd /stable-diffusion-webui/models/insightface && \
-#     wget https://github.com/facefusion/facefusion-assets/releases/download/models/inswapper_128.onnx
-
-# # Configure ReActor to use the GPU instead of the CPU
-# RUN echo "CUDA" > /stable-diffusion-webui/extensions/sd-webui-reactor/last_device.txt
-
-# # Install CivitAI Model Downloader
-# # RUN git clone --depth=1 https://github.com/ashleykleynhans/civitai-downloader.git && \
-# #     mv civitai-downloader/download.sh /usr/local/bin/download-model && \
-# #     chmod +x /usr/local/bin/download-model
-
-# # Copy Stable Diffusion Web UI config files
-# # COPY a1111/relauncher.py a1111/webui-user.sh a1111/config.json a1111/ui-config.json /stable-diffusion-webui/
-# COPY a1111/webui-user.sh a1111/config.json a1111/ui-config.json /stable-diffusion-webui/
-
 # ADD SDXL styles.csv
 ADD https://raw.githubusercontent.com/Douleb/SDXL-750-Styles-GPT4-/main/styles.csv /config/styles.csv
 
@@ -182,6 +81,20 @@ RUN rm -f /etc/ssh/ssh_host_*
 
 WORKDIR /
 
+ENV INSTALL_ROOT=/workspace/apps
+ENV KOHYA_ROOT="${INSTALL_ROOT}/kohya_ss"
+ENV INVOKEAI_ROOT="${INSTALL_ROOT}/invokeai"
+
+ENV WEBUI_VERSION=v1.6.0
+ENV DREAMBOOTH_COMMIT=cf086c536b141fc522ff11f6cffc8b7b12da04b9
+ENV KOHYA_VERSION=v22.2.2
+ENV INVOKEAI_VERSION=v3.4.0post2
+
+ENV DISABLE_MODEL_DOWNLOAD=false
+ENV DISABLE_TRAINING_ASSET_DOWNLOAD=false
+ENV SHUTDOWN_AFTER_PROVISION=false
+ENV DISABLE_AUTOLAUNCH=false
+
 # Copy the scripts
 COPY --chmod=755 scripts/* ./
 COPY model-download-aria2.txt /

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -7,6 +7,7 @@ services:
       VS_SERVER_PASSWORD: testpass
       # DISABLE_MODEL_DOWNLOAD: "true"
       DISABLE_TRAINING_ASSET_DOWNLOAD: "true"
+      # SHUTDOWN_AFTER_PROVISION: "true"
     volumes:
       - /mnt/c/Users/Hemul/StableDiffusion/workspace:/workspace
     ports:

diff --git a/invokeai/invokeai.yaml b/invokeai/invokeai.yaml
@@ -23,7 +23,7 @@ InvokeAI:
     models_dir: models
     legacy_conf_dir: configs/stable-diffusion
     db_dir: databases
-    outdir: /invokeai/outputs
+    outdir: outputs
     use_memory_db: false
     custom_nodes_dir: nodes
   Logging:

diff --git a/scripts/pre_start.sh b/scripts/pre_start.sh
@@ -1,25 +1,25 @@
 #!/usr/bin/env bash
+set -eu
+
 export PYTHONUNBUFFERED=1
 
 # Configure accelerate
 echo "Configuring accelerate..."
 mkdir -p /root/.cache/huggingface/accelerate
-mv /accelerate.yaml /root/.cache/huggingface/accelerate/default_config.yaml
+cp /accelerate.yaml /root/.cache/huggingface/accelerate/default_config.yaml
 
 echo "Setting the app install root to ${INSTALL_ROOT}"
 mkdir -p ${INSTALL_ROOT}
-export KOHYA_ROOT="${INSTALL_ROOT}/kohya_ss"
-export INVOKEAI_ROOT="${INSTALL_ROOT}/invokeai"
-export A1111_ROOT="${INSTALL_ROOT}/a1111"
-
-[ -d ${INVOKEAI_ROOT} ] || /install_invoke.sh
-[ -d ${KOHYA_ROOT} ]    || /install_kohya.sh
 
-ln -fs /workspace/outputs/invokeai  ${INVOKEAI_ROOT}/outputs
+echo "Starting app provision in parallel..."
+/provision_invokeai.sh &
+/provision_kohya.sh &
+wait
+echo "All app installs complete"
 
 mkdir -p /workspace/logs
 
-if [[ ! ${DISABLE_MODEL_DOWNLOAD} ]]; then
+if [ "${DISABLE_MODEL_DOWNLOAD}" != true ]; then
     echo "Downloading missing shared models..."
     mkdir -p /workspace/models/main
     aria2c -i /model-download-aria2.txt -j 4 -c
@@ -33,7 +33,7 @@ if [[ ! ${DISABLE_MODEL_DOWNLOAD} ]]; then
     ln -fs /workspace/models/main/sd_xl_refiner_1.0_0.9vae.safetensors  ${INVOKEAI_ROOT}/autoimport/sd_xl_refiner_1.0_0.9vae.safetensors
 fi
 
-if [[ ! ${DISABLE_TRAINING_ASSET_DOWNLOAD} ]] && [ ! -d /workspace/training-assets ]; then
+if [ "${DISABLE_TRAINING_ASSET_DOWNLOAD}" != true ] && [ ! -d /workspace/training-assets ]; then
     echo "Downloading training assets..."
     mkdir -p /workspace/training-assets
     [ -f /workspace/training-assets.tar.gz ] || \

diff --git a/scripts/install_invokeai.sh → scripts/provision_invokeai.sh b/scripts/install_invokeai.sh → scripts/provision_invokeai.sh
@@ -1,19 +1,24 @@
 #!/usr/bin/env bash
 set -eu
 
-rm -rf ${INVOKEAI_ROOT}
+[ -d ${INVOKEAI_ROOT} ] || git clone https://github.com/invoke-ai/InvokeAI.git ${INVOKEAI_ROOT}
 
-# Install InvokeAI
-git clone https://github.com/invoke-ai/InvokeAI.git ${INVOKEAI_ROOT}
 cd ${INVOKEAI_ROOT}
 
+git fetch --tags
 git checkout ${INVOKEAI_VERSION}
-python3 -m venv --system-site-packages venv
+
+if [ -f install_complete ]; then 
+    echo "install_complete flag file found, skipping InvokeAI install..."
+    exit
+fi
+
+[ -d venv ] || python3 -m venv --system-site-packages venv
+
 source venv/bin/activate
 pip3 install "InvokeAI[xformers]" --use-pep517 --extra-index-url https://download.pytorch.org/whl/cu121
-pip3 cache purge
 
-cp /config/invokeai/invokeai.yaml ./invokeai.yaml
+[ -f invokeai.yaml ] || cp /config/invokeai/invokeai.yaml ./invokeai.yaml
 
 echo "Configuring InvokeAI..."
 invokeai-configure --root ${INVOKEAI_ROOT} --yes --default_only --skip-sd-weights
@@ -23,6 +28,5 @@ invokeai-model-install --root ${INVOKEAI_ROOT} --yes --add \
     diffusers/controlnet-canny-sdxl-1.0 \
     diffusers/controlnet-depth-sdxl-1.0 \
     madebyollin/sdxl-vae-fp16-fix
-touch /workspace/invoke/models_fetched
-
-deactivate
+deactivate
+touch install_complete
diff --git a/scripts/install_kohya.sh → scripts/provision_kohya.sh b/scripts/install_kohya.sh → scripts/provision_kohya.sh
@@ -1,18 +1,22 @@
 #!/usr/bin/env bash
 set -eu
 
-rm -rf ${KOHYA_ROOT}
-
 # Install Kohya_ss
-git clone https://github.com/bmaltais/kohya_ss.git ${KOHYA_ROOT}
+[ -d ${KOHYA_ROOT} ] || git clone https://github.com/bmaltais/kohya_ss.git ${KOHYA_ROOT}
 
 cd ${KOHYA_ROOT}
 
-cp /config/kohya_ss/requirements* ./
-
+git fetch --tags
 git checkout ${KOHYA_VERSION}
 
-python3 -m venv --system-site-packages venv
+if [ -f install_complete ]; then 
+    echo "install_complete flag file found, skipping Kohya install..."
+    exit
+fi
+
+cp /config/kohya_ss/requirements* ./
+
+[ -d venv ] || python3 -m venv --system-site-packages venv
 source venv/bin/activate
 
 pip3 install --no-cache-dir torch==2.0.1 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
@@ -25,5 +29,5 @@ pip3 install --no-cache-dir xformers==0.0.22 \
     tensorrt
 pip3 install -r requirements.txt
 pip3 install .
-pip3 cache purge
 deactivate
+touch install_complete
diff --git a/scripts/start.sh b/scripts/start.sh
@@ -71,20 +71,22 @@ start_vsserver () {
 
 execute_script "/pre_start.sh" "Running pre-start script..."
 
+
+if [ "${SHUTDOWN_AFTER_PROVISION}" = true ]; then
+    echo "Provisioning complete, shutting down..."
+    [ -z ${RUNPOD_POD_ID} ] || runpodctl remove pod ${RUNPOD_POD_ID}
+    exit
+fi 
+
 echo "Pod Started"
 
 echo "Starting services..."
-if [[ ! ${DISABLE_AUTOLAUNCH} ]]; then
+if [ "${DISABLE_AUTOLAUNCH}" != true ]; then
     # /start_a1111.sh
     /start_kohya.sh
     /start_invokeai.sh
 fi
 
-if [ ${ENABLE_TENSORBOARD} ];
-then
-    /start_tensorboard.sh
-fi
-
 setup_ssh
 start_vsserver
 export_env_vars

diff --git a/scripts/start_a1111.sh b/scripts/start_a1111.sh
@@ -3,7 +3,7 @@ echo "Starting Stable Diffusion Web UI"
 cd /stable-diffusion-webui
 source ./venv/bin/activate
 
-nohup ./webui.sh -f > /workspace/logs/a1111.log 2>&1 &
+nohup ./webui.sh -f >> /workspace/logs/a1111.log 2>&1 &
 echo $! > /a1111.pid
 echo "Stable Diffusion Web UI started"
 echo "Log file: /workspace/logs/a1111.log"

diff --git a/scripts/start_invokeai.sh b/scripts/start_invokeai.sh
@@ -3,7 +3,7 @@ echo "Starting InvokeAI..."
 source ${INVOKEAI_ROOT}/venv/bin/activate
 
 nohup ${INVOKEAI_ROOT}/scripts/invokeai-web.py \
-    > /workspace/logs/invokeai.log 2>&1 &           
+    >> /workspace/logs/invokeai.log 2>&1 &           
 echo $! > /invokeai.pid
 echo "Started InvokeAI"
 echo "Log file: /workspace/logs/invokeai.log"

diff --git a/scripts/start_kohya.sh b/scripts/start_kohya.sh
@@ -1,6 +1,6 @@
 echo "Starting Kohya_ss Web UI"
 cd ${KOHYA_ROOT}
-nohup ./gui.sh --listen 0.0.0.0 --server_port 3010 --headless > /workspace/logs/kohya_ss.log 2>&1 &
+nohup ./gui.sh --listen 0.0.0.0 --server_port 3010 --headless >> /workspace/logs/kohya_ss.log 2>&1 &
 echo $! > /kohya_ss.pid
 echo "Kohya_ss started"
 echo "Log file: /workspace/logs/kohya_ss.log"

diff --git a/scripts/start_tensorboard.sh b/scripts/start_tensorboard.sh
@@ -10,7 +10,8 @@ if [[ ! -L /workspace/logs/ti ]]; then
     ln -s /workspace/stable-diffusion-webui/textual_inversion /workspace/logs/ti
 fi
 source /workspace/venv/bin/activate
-nohup tensorboard --logdir=/workspace/logs --port=6066 --host=0.0.0.0 > /workspace/logs/tensorboard.log 2>&1 &
+nohup tensorboard --logdir=/workspace/logs --port=6066 --host=0.0.0.0 \
+    >> /workspace/logs/tensorboard.log 2>&1 &
 echo $! > /tensorboard.pid
 deactivate
 echo "Tensorboard Started"

diff --git a/scripts/start_vs_server.sh b/scripts/start_vs_server.sh
@@ -2,9 +2,8 @@
 echo "Starting VS Server..."
 PASSWORD=${VS_SERVER_PASSWORD} \
 nohup code-server \
-    --verbose \
     --bind-addr 0.0.0.0:8080 \
-    > /workspace/logs/vs-server.log 2>&1 &           
+    >> /workspace/logs/vs-server.log 2>&1 &           
 echo $! > /vs-server.pid
 echo "Started VS Server"
 echo "Log file: /workspace/logs/vs-server.log"