Add multi stage build for demo_pipeline (#84)

Signed-off-by: romanodanilo <[email protected]>

Signed-off-by: romanodanilo <[email protected]>

.github/workflows/build-on-change-linux-bare.yaml

@@ -99,8 +99,8 @@ jobs:
         if: github.event_name == 'pull_request'
         run: |
           mv build out_build
-          cp -r /home/$(whoami)/qc-build build
-          cp out_build/examples/checker_bundle_example/DemoCheckerBundle build/bin/
+          cp -r /home/$(whoami)/qc-build/bin bin
+          cp out_build/examples/checker_bundle_example/DemoCheckerBundle bin/
           cd runtime
           python3 -m pip install -r requirements.txt
           python3 -m pytest -rA > runtime_test.log

.github/workflows/build-on-change-windows.yaml

@@ -139,8 +139,8 @@ jobs:
           Write-Output "Starting runtime tests..."
 
           Rename-Item -path "$env:WORKING_PATH\qc-framework\qc-framework\build" -NewName "$env:WORKING_PATH\qc-framework\qc-framework\out_build"
-          Copy-Item -Path "$env:WORKING_PATH\QC-Framework-Out" -Destination "$env:WORKING_PATH\qc-framework\qc-framework\build" -Recurse
-          Copy-Item -Path "$env:WORKING_PATH\qc-framework\qc-framework\out_build\examples\checker_bundle_example\Release\DemoCheckerBundle.exe" -Destination "$env:WORKING_PATH\qc-framework\qc-framework\build\bin"
+          Copy-Item -Path "$env:WORKING_PATH\QC-Framework-Out\bin" -Destination "$env:WORKING_PATH\qc-framework\qc-framework\bin" -Recurse
+          Copy-Item -Path "$env:WORKING_PATH\qc-framework\qc-framework\out_build\examples\checker_bundle_example\Release\DemoCheckerBundle.exe" -Destination "$env:WORKING_PATH\qc-framework\qc-framework\bin"
 
           cd "$env:WORKING_PATH\qc-framework\qc-framework\runtime"
           python3 -m pip install -r requirements.txt

demo_pipeline/run_pipeline.sh

@@ -7,10 +7,10 @@ python3 /app/demo_pipeline/configuration_generator.py
 
 python3 /app/framework/runtime/runtime/runtime.py \
     --config "/tmp/generated_config/config.xml" \
-    --install_dir "/app/framework/build/bin" \
+    --install_dir "/app/framework/bin" \
     --schema_dir "/app/framework/doc/schema"
 
 rm -rf /out/$INPUT_FILENAME
 mkdir -p /out/$INPUT_FILENAME
-cp /app/framework/build/bin/*.xqar /out/$INPUT_FILENAME
-cp /app/framework/build/bin/*.txt /out/$INPUT_FILENAME
+cp /app/framework/bin/*.xqar /out/$INPUT_FILENAME
+cp /app/framework/bin/*.txt /out/$INPUT_FILENAME

demo_pipeline/xodrBundle

@@ -9,4 +9,4 @@ readonly CONFIG_FILE=$1
 source /app/opendrive-venv/bin/activate
 cd /app/qc-opendrive
 python main.py -c $CONFIG_FILE
-cp *.xqar /app/framework/build/bin/
+cp *.xqar /app/framework/bin/

demo_pipeline/xoscBundle

@@ -9,4 +9,4 @@ readonly CONFIG_FILE=$1
 source /app/openscenario-venv/bin/activate
 cd /app/qc-openscenarioxml
 python main.py -c $CONFIG_FILE
-cp *.xqar /app/framework/build/bin/
+cp *.xqar /app/framework/bin/

docker/Dockerfile.linux

@@ -18,11 +18,15 @@ RUN echo "Installing Dependencies..." && \
     qtbase5-dev \
     libqt5xmlpatterns5-dev \
     libxerces-c-dev \
-    pkg-config && \
+    pkg-config \
+    python3.10-venv \
+    git && \
     echo "Dependencies installed."
 
 RUN mkdir -p /app/framework
 
+WORKDIR /app
+
 COPY examples /app/framework/examples
 COPY include /app/framework/include
 COPY scripts /app/framework/scripts
@@ -33,6 +37,7 @@ COPY CMakeLists.txt /app/framework/CMakeLists.txt
 COPY version /app/framework/version
 COPY licenses /app/framework/licenses
 COPY runtime /app/framework/runtime
+COPY demo_pipeline /app/demo_pipeline
 
 RUN echo "Building framework..." && \
     cd /app/framework && \
@@ -46,9 +51,38 @@ RUN echo "Building framework..." && \
     echo "Done."
 
 
+# Clone, configure venv, install by copying bash script to install dir
+RUN git clone --single-branch --branch develop https://github.com/asam-ev/qc-openscenarioxml.git && \
+    python3 -m venv openscenario-venv && \
+    source openscenario-venv/bin/activate && \
+    python3 -m pip install --no-cache-dir -r qc-openscenarioxml/requirements.txt && \
+    chmod +x /app/demo_pipeline/xoscBundle && \
+    cp /app/demo_pipeline/xoscBundle /home/root/qc-build/bin/xoscBundle && \
+    rm -rf /app/qc-openscenarioxml/.git
+
+# Clone, configure venv, install by copying bash script to install dir
+RUN git clone --single-branch --branch develop https://github.com/asam-ev/qc-opendrive.git && \
+    python3 -m venv opendrive-venv && \
+    source opendrive-venv/bin/activate && \
+    python3 -m pip install --no-cache-dir -r qc-opendrive/requirements.txt && \
+    chmod +x /app/demo_pipeline/xodrBundle && \
+    cp /app/demo_pipeline/xodrBundle /home/root/qc-build/bin/xodrBundle && \
+    rm -rf /app/qc-opendrive/.git
+
+
+RUN python3 -m venv demo-pipeline-venv && \
+    source demo-pipeline-venv/bin/activate && \
+    python3 -m pip install --no-cache-dir -r /app/demo_pipeline/requirements.txt
+
+RUN python3 -m venv runtime-venv && \
+    source runtime-venv/bin/activate && \
+    python3 -m pip install --no-cache-dir -r /app/framework/runtime/requirements.txt
+
+
 # Runtime stage
-FROM python:3.11.9-slim-bookworm  as runtime_test
+FROM ubuntu:22.04 as runtime_test
 
+# Dependancies installation currently required by ResultPooling and TextReport modules
 RUN echo "Installing Qt..." && \
     apt update && apt install -y \
     qtbase5-dev \
@@ -58,51 +92,57 @@ RUN echo "Installing Qt..." && \
     echo "Dependencies installed."
 
 RUN  mkdir -p /app
-COPY runtime/requirements.txt /app/
-RUN pip install -r /app/requirements.txt
+
 # For testing files
 COPY --from=framework_builder /app/framework /app/framework
 # Copy install directory in runtime image
-COPY --from=framework_builder /home/root/qc-build /app/framework/build
+COPY --from=framework_builder /home/root/qc-build/bin /app/framework/bin
+COPY --from=framework_builder /app/framework/build/test /app/framework/test
 # @NOTE this is just because DemoCheckerBundle is not installed by default
-COPY --from=framework_builder /app/framework/build/examples/checker_bundle_example/DemoCheckerBundle /app/framework/build/bin/
-ENV PYTHONUNBUFFERED=1
+COPY --from=framework_builder /app/framework/build/examples/checker_bundle_example/DemoCheckerBundle /app/framework/bin/
+# Virtual envs
+COPY --from=framework_builder /app/runtime-venv /app/runtime-venv
+
 WORKDIR /app/framework/runtime/
-CMD python3 -m pytest -rA > runtime_test.log && cp /app/framework/runtime/runtime_test.log /out/runtime_test.log
 
+SHELL ["/bin/bash", "-c"]
+
+CMD source /app/runtime-venv/bin/activate && python3 -m pytest -rA > runtime_test.log && cp /app/framework/runtime/runtime_test.log /out/runtime_test.log
 
 # Runtime stage
 FROM framework_builder as unit_test
 CMD ctest --test-dir /app/framework/build -C Release && cp /app/framework/build/Testing/Temporary/LastTest.log /out/
 
 
-FROM runtime_test as demo_pipeline
+FROM ubuntu:22.04 as demo_pipeline
 
-SHELL ["/bin/bash", "-c"]
+# Dependancies installation currently required by ResultPooling and TextReport modules
+RUN echo "Installing Qt..." && \
+    apt update && apt install -y \
+    qtbase5-dev \
+    libqt5xmlpatterns5-dev \
+    libxerces-c-dev \
+    pkg-config && \
+    echo "Dependencies installed."
 
-RUN apt install -y git
+RUN  mkdir -p /app
 
-WORKDIR /app
+# Copy install directory in runtime image
+COPY --from=framework_builder /home/root/qc-build/bin /app/framework/bin
 
-# Clone, configure venv, install by copying bash script to install dir
-RUN git clone --single-branch --branch develop https://github.com/asam-ev/qc-openscenarioxml.git && \
-    # Create a virtual environment
-    python3.11 -m venv openscenario-venv && \
-    # Activate the virtual environment and install required Python packages
-    source openscenario-venv/bin/activate && \
-    pip install --no-cache-dir -r qc-openscenarioxml/requirements.txt
+# Virtual envs
+COPY --from=framework_builder /app/demo-pipeline-venv /app/demo-pipeline-venv
+COPY --from=framework_builder /app/opendrive-venv /app/opendrive-venv
+COPY --from=framework_builder /app/openscenario-venv /app/openscenario-venv
 
-# Clone, configure venv, install by copying bash script to install dir
-RUN git clone --single-branch --branch develop https://github.com/asam-ev/qc-opendrive.git && \
-    # Create a virtual environment
-    python3.11 -m venv opendrive-venv && \
-    # Activate the virtual environment and install required Python packages
-    source opendrive-venv/bin/activate && \
-    pip install --no-cache-dir -r qc-opendrive/requirements.txt
+# Source files
+COPY --from=framework_builder /app/demo_pipeline /app/demo_pipeline
+COPY --from=framework_builder /app/framework /app/framework
 
-COPY demo_pipeline /app/demo_pipeline
-RUN chmod +x /app/demo_pipeline/xoscBundle && cp /app/demo_pipeline/xoscBundle /app/framework/build/bin/
-RUN chmod +x /app/demo_pipeline/xodrBundle && cp /app/demo_pipeline/xodrBundle /app/framework/build/bin/
+# Cloned repositories
+COPY --from=framework_builder /app/qc-openscenarioxml /app/qc-openscenarioxml
+COPY --from=framework_builder /app/qc-opendrive /app/qc-opendrive
+
+SHELL ["/bin/bash", "-c"]
 
-RUN pip install -r /app/demo_pipeline/requirements.txt
-CMD /app/demo_pipeline/run_pipeline.sh
+CMD source /app/demo-pipeline-venv/bin/activate && /app/demo_pipeline/run_pipeline.sh

runtime/tests/test_runtime.py

@@ -58,15 +58,15 @@ def check_node_exists(xml_file: str, node_name: str) -> bool:
 def test_runtime_execution():
 
     start_wd = os.getcwd()
-    install_dir = os.path.join("..", "build", "bin")
+    install_dir = os.path.join("..", "bin")
     os.chdir(install_dir)
 
     config_xml = os.path.join(
-        "..", "..", "runtime", "tests", "test_data", "DemoCheckerBundle_config.xml"
+        "..", "runtime", "tests", "test_data", "DemoCheckerBundle_config.xml"
     )
 
-    schema_dir = os.path.join("..", "..", "doc", "schema")
-    runtime_script = os.path.join("..", "..", "runtime", "runtime", "runtime.py")
+    schema_dir = os.path.join("..", "doc", "schema")
+    runtime_script = os.path.join("..", "runtime", "runtime", "runtime.py")
 
     process = subprocess.Popen(
         f"python3 {runtime_script} --config={config_xml} --install_dir={os.getcwd()} --schema_dir={schema_dir}",
@@ -101,15 +101,13 @@ def test_runtime_execution():
 def test_3steps_config():
 
     start_wd = os.getcwd()
-    install_dir = os.path.join("..", "build", "bin")
+    install_dir = os.path.join("..", "bin")
     os.chdir(install_dir)
 
-    config_xml = os.path.join(
-        "..", "..", "runtime", "tests", "test_data", "3steps_config.xml"
-    )
+    config_xml = os.path.join("..", "runtime", "tests", "test_data", "3steps_config.xml")
 
-    schema_dir = os.path.join("..", "..", "doc", "schema")
-    runtime_script = os.path.join("..", "..", "runtime", "runtime", "runtime.py")
+    schema_dir = os.path.join("..", "doc", "schema")
+    runtime_script = os.path.join("..", "runtime", "runtime", "runtime.py")
 
     process = subprocess.Popen(
         f"python3 {runtime_script} --config={config_xml} --install_dir={os.getcwd()} --schema_dir={schema_dir}",