Enabled golden image validation on GitHub CI (#50)

DiligentGraphics · Aug 9, 2022 · 30c0e75 · 30c0e75
1 parent 881e626
commit 30c0e75
Show file tree

Hide file tree

Showing 52 changed files with 40 additions and 222 deletions.
diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml
@@ -2,6 +2,12 @@ name: Win32/UWP build
 
 on: [push, pull_request]
 
+env:
+  # 'capture'        - generate golden images
+  # 'compare'        - render and compare with the references
+  # 'compare_update' - first compare with the references and then update them
+  test_mode: compare_update
+
 jobs:
   precheks:
     name:    Win10 -> Pre-checks
@@ -23,13 +29,14 @@ jobs:
       shell: cmd
       working-directory: ${{github.workspace}}/DiligentSamples/BuildTools/FormatValidation
       run: validate_format_win.bat
-    
+
 
   build:
     needs:   precheks
     runs-on: windows-2022
 
     strategy:
+      fail-fast: false
       matrix:
         name:            ["Win10"]
         toolset:         ["Win32", "x64"]
@@ -85,9 +92,15 @@ jobs:
     name: Win10 -> ${{ matrix.name }}-${{ matrix.toolset }}, ${{ matrix.build_type }}
 
     steps:
-    - name: Checkout
+    - name: Checkout Code Base
       uses: DiligentGraphics/github-action/checkout@v1
 
+    - name: Checkout Test Data
+      uses: actions/checkout@v2
+      with:
+        repository: DiligentGraphics/DiligentTestData
+        path:       DiligentTestData
+
     - name: Set up build environment
       if:   success()
       uses: DiligentGraphics/github-action/setup-build-env@v1
@@ -108,3 +121,28 @@ jobs:
     - name: Build
       if:   success()
       uses: DiligentGraphics/github-action/build@v1
+
+    - name: Run Sample Tests D3D11
+      working-directory: ${{github.workspace}}/DiligentTestData/GoldenImages
+      if:   ${{ success() && (matrix.name == 'Win10' || matrix.name == 'Win8.1') }}
+      shell: cmd
+      run: |
+        :: Run Sample Tests D3D11
+        ProcessGoldenImages.bat "$DILIGENT_BUILD_DIR" ${{matrix.build_type}} ${{matrix.test_mode}} "--mode d3d11 --adapter sw"
+
+    - name: Run Sample Tests D3D12
+      working-directory: ${{github.workspace}}/DiligentTestData/GoldenImages
+      if:   ${{ success() && matrix.name == 'Win10' }}
+      shell: cmd
+      run: |
+        :: Run Sample Tests D3D12
+        ProcessGoldenImages.bat "$DILIGENT_BUILD_DIR" ${{matrix.build_type}} ${{matrix.test_mode}} "--mode d3d12 --adapter sw"
+
+    - name: Upload Golden Images
+      uses: actions/upload-artifact@v2
+      if: ${{ matrix.name == 'Win10' || matrix.name == 'Win8.1' }}
+      with:
+        name: GoldenImages-${{ matrix.name }}-${{ matrix.toolset }}-${{ matrix.build_type }}
+        path: |
+          ${{github.workspace}}/DiligentTestData/GoldenImages
+        retention-days: 90
diff --git a/BuildTools/Scripts/appveyor/run_tests.bat b/BuildTools/Scripts/appveyor/run_tests.bat
diff --git a/Tests/GoldenImages/ProcessGoldenImages.bat b/Tests/GoldenImages/ProcessGoldenImages.bat
diff --git a/Tests/GoldenImages/Samples/Atmosphere/Atmosphere_gi_d3d11.png b/Tests/GoldenImages/Samples/Atmosphere/Atmosphere_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/Atmosphere/Atmosphere_gi_d3d12.png b/Tests/GoldenImages/Samples/Atmosphere/Atmosphere_gi_d3d12.png
diff --git a/Tests/GoldenImages/Samples/GLTFViewer/GLTFViewer_gi_d3d11.png b/Tests/GoldenImages/Samples/GLTFViewer/GLTFViewer_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/GLTFViewer/GLTFViewer_gi_d3d12.png b/Tests/GoldenImages/Samples/GLTFViewer/GLTFViewer_gi_d3d12.png
diff --git a/Tests/GoldenImages/Samples/ImguiDemo/ImguiDemo_gi_d3d11.png b/Tests/GoldenImages/Samples/ImguiDemo/ImguiDemo_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/ImguiDemo/ImguiDemo_gi_d3d12.png b/Tests/GoldenImages/Samples/ImguiDemo/ImguiDemo_gi_d3d12.png
diff --git a/Tests/GoldenImages/Samples/NuklearDemo/NuklearDemo_gi_d3d11.png b/Tests/GoldenImages/Samples/NuklearDemo/NuklearDemo_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/NuklearDemo/NuklearDemo_gi_d3d12.png b/Tests/GoldenImages/Samples/NuklearDemo/NuklearDemo_gi_d3d12.png
diff --git a/Tests/GoldenImages/Samples/Shadows/Shadows_gi_d3d11.png b/Tests/GoldenImages/Samples/Shadows/Shadows_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/Shadows/Shadows_gi_d3d12.png b/Tests/GoldenImages/Samples/Shadows/Shadows_gi_d3d12.png
diff --git a/...Images/Tutorials/Tutorial01_HelloTriangle/Tutorial01_HelloTriangle_gi_d3d11.png b/...Images/Tutorials/Tutorial01_HelloTriangle/Tutorial01_HelloTriangle_gi_d3d11.png
diff --git a/...Images/Tutorials/Tutorial01_HelloTriangle/Tutorial01_HelloTriangle_gi_d3d12.png b/...Images/Tutorials/Tutorial01_HelloTriangle/Tutorial01_HelloTriangle_gi_d3d12.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial02_Cube/Tutorial02_Cube_gi_d3d11.png b/Tests/GoldenImages/Tutorials/Tutorial02_Cube/Tutorial02_Cube_gi_d3d11.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial02_Cube/Tutorial02_Cube_gi_d3d12.png b/Tests/GoldenImages/Tutorials/Tutorial02_Cube/Tutorial02_Cube_gi_d3d12.png
diff --git a/...ldenImages/Tutorials/Tutorial03_Texturing-C/Tutorial03_Texturing-C_gi_d3d11.png b/...ldenImages/Tutorials/Tutorial03_Texturing-C/Tutorial03_Texturing-C_gi_d3d11.png
diff --git a/...ldenImages/Tutorials/Tutorial03_Texturing-C/Tutorial03_Texturing-C_gi_d3d12.png b/...ldenImages/Tutorials/Tutorial03_Texturing-C/Tutorial03_Texturing-C_gi_d3d12.png
diff --git a/...s/GoldenImages/Tutorials/Tutorial03_Texturing/Tutorial03_Texturing_gi_d3d11.png b/...s/GoldenImages/Tutorials/Tutorial03_Texturing/Tutorial03_Texturing_gi_d3d11.png
diff --git a/...s/GoldenImages/Tutorials/Tutorial03_Texturing/Tutorial03_Texturing_gi_d3d12.png b/...s/GoldenImages/Tutorials/Tutorial03_Texturing/Tutorial03_Texturing_gi_d3d12.png
diff --git a/...GoldenImages/Tutorials/Tutorial04_Instancing/Tutorial04_Instancing_gi_d3d11.png b/...GoldenImages/Tutorials/Tutorial04_Instancing/Tutorial04_Instancing_gi_d3d11.png
diff --git a/...GoldenImages/Tutorials/Tutorial04_Instancing/Tutorial04_Instancing_gi_d3d12.png b/...GoldenImages/Tutorials/Tutorial04_Instancing/Tutorial04_Instancing_gi_d3d12.png
diff --git a/...enImages/Tutorials/Tutorial05_TextureArray/Tutorial05_TextureArray_gi_d3d11.png b/...enImages/Tutorials/Tutorial05_TextureArray/Tutorial05_TextureArray_gi_d3d11.png
diff --git a/...enImages/Tutorials/Tutorial05_TextureArray/Tutorial05_TextureArray_gi_d3d12.png b/...enImages/Tutorials/Tutorial05_TextureArray/Tutorial05_TextureArray_gi_d3d12.png
diff --git a/...ages/Tutorials/Tutorial06_Multithreading/Tutorial06_Multithreading_gi_d3d11.png b/...ages/Tutorials/Tutorial06_Multithreading/Tutorial06_Multithreading_gi_d3d11.png
diff --git a/...ages/Tutorials/Tutorial06_Multithreading/Tutorial06_Multithreading_gi_d3d12.png b/...ages/Tutorials/Tutorial06_Multithreading/Tutorial06_Multithreading_gi_d3d12.png
diff --git a/...ages/Tutorials/Tutorial07_GeometryShader/Tutorial07_GeometryShader_gi_d3d11.png b/...ages/Tutorials/Tutorial07_GeometryShader/Tutorial07_GeometryShader_gi_d3d11.png
diff --git a/...ages/Tutorials/Tutorial07_GeometryShader/Tutorial07_GeometryShader_gi_d3d12.png b/...ages/Tutorials/Tutorial07_GeometryShader/Tutorial07_GeometryShader_gi_d3d12.png
diff --git a/...enImages/Tutorials/Tutorial08_Tessellation/Tutorial08_Tessellation_gi_d3d11.png b/...enImages/Tutorials/Tutorial08_Tessellation/Tutorial08_Tessellation_gi_d3d11.png
diff --git a/...enImages/Tutorials/Tutorial08_Tessellation/Tutorial08_Tessellation_gi_d3d12.png b/...enImages/Tutorials/Tutorial08_Tessellation/Tutorial08_Tessellation_gi_d3d12.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial09_Quads/Tutorial09_Quads_gi_d3d11.png b/Tests/GoldenImages/Tutorials/Tutorial09_Quads/Tutorial09_Quads_gi_d3d11.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial09_Quads/Tutorial09_Quads_gi_d3d12.png b/Tests/GoldenImages/Tutorials/Tutorial09_Quads/Tutorial09_Quads_gi_d3d12.png
diff --git a/...Images/Tutorials/Tutorial10_DataStreaming/Tutorial10_DataStreaming_gi_d3d11.png b/...Images/Tutorials/Tutorial10_DataStreaming/Tutorial10_DataStreaming_gi_d3d11.png
diff --git a/...Images/Tutorials/Tutorial10_DataStreaming/Tutorial10_DataStreaming_gi_d3d12.png b/...Images/Tutorials/Tutorial10_DataStreaming/Tutorial10_DataStreaming_gi_d3d12.png
diff --git a/...es/Tutorials/Tutorial11_ResourceUpdates/Tutorial11_ResourceUpdates_gi_d3d11.png b/...es/Tutorials/Tutorial11_ResourceUpdates/Tutorial11_ResourceUpdates_gi_d3d11.png
diff --git a/...es/Tutorials/Tutorial11_ResourceUpdates/Tutorial11_ResourceUpdates_gi_d3d12.png b/...es/Tutorials/Tutorial11_ResourceUpdates/Tutorial11_ResourceUpdates_gi_d3d12.png
diff --git a/...enImages/Tutorials/Tutorial12_RenderTarget/Tutorial12_RenderTarget_gi_d3d11.png b/...enImages/Tutorials/Tutorial12_RenderTarget/Tutorial12_RenderTarget_gi_d3d11.png
diff --git a/...enImages/Tutorials/Tutorial12_RenderTarget/Tutorial12_RenderTarget_gi_d3d12.png b/...enImages/Tutorials/Tutorial12_RenderTarget/Tutorial12_RenderTarget_gi_d3d12.png
diff --git a/...s/GoldenImages/Tutorials/Tutorial13_ShadowMap/Tutorial13_ShadowMap_gi_d3d11.png b/...s/GoldenImages/Tutorials/Tutorial13_ShadowMap/Tutorial13_ShadowMap_gi_d3d11.png
diff --git a/...s/GoldenImages/Tutorials/Tutorial13_ShadowMap/Tutorial13_ShadowMap_gi_d3d12.png b/...s/GoldenImages/Tutorials/Tutorial13_ShadowMap/Tutorial13_ShadowMap_gi_d3d12.png
diff --git a/...Images/Tutorials/Tutorial14_ComputeShader/Tutorial14_ComputeShader_gi_d3d11.png b/...Images/Tutorials/Tutorial14_ComputeShader/Tutorial14_ComputeShader_gi_d3d11.png
diff --git a/...Images/Tutorials/Tutorial14_ComputeShader/Tutorial14_ComputeShader_gi_d3d12.png b/...Images/Tutorials/Tutorial14_ComputeShader/Tutorial14_ComputeShader_gi_d3d12.png
diff --git a/...utorials/Tutorial16_BindlessResources/Tutorial16_BindlessResources_gi_d3d11.png b/...utorials/Tutorial16_BindlessResources/Tutorial16_BindlessResources_gi_d3d11.png
diff --git a/...utorials/Tutorial16_BindlessResources/Tutorial16_BindlessResources_gi_d3d12.png b/...utorials/Tutorial16_BindlessResources/Tutorial16_BindlessResources_gi_d3d12.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial17_MSAA/Tutorial17_MSAA_gi_d3d11.png b/Tests/GoldenImages/Tutorials/Tutorial17_MSAA/Tutorial17_MSAA_gi_d3d11.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial17_MSAA/Tutorial17_MSAA_gi_d3d12.png b/Tests/GoldenImages/Tutorials/Tutorial17_MSAA/Tutorial17_MSAA_gi_d3d12.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial18_Queries/Tutorial18_Queries_gi_d3d11.png b/Tests/GoldenImages/Tutorials/Tutorial18_Queries/Tutorial18_Queries_gi_d3d11.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial18_Queries/Tutorial18_Queries_gi_d3d12.png b/Tests/GoldenImages/Tutorials/Tutorial18_Queries/Tutorial18_Queries_gi_d3d12.png
diff --git a/...enImages/Tutorials/Tutorial19_RenderPasses/Tutorial19_RenderPasses_gi_d3d11.png b/...enImages/Tutorials/Tutorial19_RenderPasses/Tutorial19_RenderPasses_gi_d3d11.png
diff --git a/...enImages/Tutorials/Tutorial19_RenderPasses/Tutorial19_RenderPasses_gi_d3d12.png b/...enImages/Tutorials/Tutorial19_RenderPasses/Tutorial19_RenderPasses_gi_d3d12.png
diff --git a/appveyor.yml b/appveyor.yml
@@ -10,11 +10,6 @@ image:
 - Visual Studio 2019
 
 environment:
-  # use 'capture' to generate golden images and upload them as build artifacts
-  # use 'compare' to render and compare with the golden references
-  # use 'compare_update' to first compare with the references and then update them
-  test_mode: compare_update
-
   matrix:
     - platform: x64
       configuration: Debug
@@ -52,15 +47,3 @@ build:
   project: c:\projects\build\DiligentSamples_Test.sln
   verbosity: minimal
   parallel: true
-
-test_script:
-  - 'cd %APPVEYOR_BUILD_FOLDER%\Tests\GoldenImages'
-  - '%APPVEYOR_BUILD_FOLDER%\BuildTools\Scripts\appveyor\run_tests.bat c:\projects\build %test_mode%'
-
-after_test:
-  - 'cd %APPVEYOR_BUILD_FOLDER%'
-  - 7z a golden_images-"%CONFIGURATION%".zip .\Tests\GoldenImages\*\*
-
-artifacts:
-  - path: \golden_images-*.zip
-    name: golden_images-zip