add Azure-SDK-based stream-object

.github/workflows/cmake.yml

@@ -5,6 +5,7 @@ on:
     branches: ["main"]
   pull_request:
     branches: ["main"]
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -23,14 +24,16 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies (Windows)
         if: ${{ (matrix.OS == 'windows-latest') }}
         # on Windows, we rely on vcpkg to pull in dependencies
         shell: bash
         run: |
-          vcpkg install rapidjson 'curl[ssl]' --triplet x64-windows
+          vcpkg install azure-storage-blobs-cpp:x64-windows-static
+          vcpkg install azure-identity-cpp:x64-windows-static
+          vcpkg install rapidjson 'curl[ssl]' --triplet x64-windows-static
 
       - name: Install dependencies (Linux)
         if: ${{ (matrix.OS == 'ubuntu-latest') }}
@@ -42,49 +45,115 @@ jobs:
           sudo apt-get install libfreetype6-dev -y
           sudo apt-get install rapidjson-dev -y
           sudo apt-get install libssl-dev -y
+          vcpkg install azure-storage-blobs-cpp azure-identity-cpp
+
+      - name: Install Azurite (for Azure SDK based stream tests)
+        shell: bash
+        run: |
+          npm install --location=global azurite
 
       - name: Configure CMake (Windows)
         if: ${{ (matrix.OS == 'windows-latest') }}
         shell: bash
         run: |
           # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
           # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-          # on Windows, we need to point CMake to the vcpkg-toolchain-file
-          cmake -B "${{github.workspace}}/build" -DCMAKE_BUILD_TYPE=${{matrix.build}} -DLIBCZI_BUILD_CZICMD=ON -DLIBCZI_BUILD_CURL_BASED_STREAM=ON -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL=ON -DCMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake"
+          # Note that we need to point CMake to the vcpkg-toolchain-file
+          cmake -B "${{github.workspace}}/build" -DCMAKE_BUILD_TYPE=${{matrix.build}} -DLIBCZI_BUILD_CZICMD=ON -DLIBCZI_BUILD_CURL_BASED_STREAM=ON -DLIBCZI_BUILD_AZURESDK_BASED_STREAM=ON -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL=ON -DCMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" -DVCPKG_TARGET_TRIPLET=x64-windows-static
 
       - name: Configure CMake (Linux)
         if: ${{ (matrix.OS == 'ubuntu-latest') }}
         shell: bash
         run: |
-          cmake -B "${{github.workspace}}/build" -DCMAKE_BUILD_TYPE=${{matrix.build}} -DLIBCZI_BUILD_CZICMD=ON -DLIBCZI_BUILD_CURL_BASED_STREAM=ON -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL=OFF
+          cmake -B "${{github.workspace}}/build" -DCMAKE_BUILD_TYPE=${{matrix.build}} -DLIBCZI_BUILD_CZICMD=ON -DLIBCZI_BUILD_CURL_BASED_STREAM=ON -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL=OFF -DLIBCZI_BUILD_AZURESDK_BASED_STREAM=ON -DCMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake"
 
       - name: Build
         # Build your program with the given configuration
-        run: cmake --build ${{github.workspace}}/build --config ${{matrix.build}}
+        run: cmake --build ${{github.workspace}}/build --config ${{matrix.build}} -j
 
       - name: Test
+        shell: bash
         working-directory: ${{github.workspace}}/build
-        # Execute tests defined by the CMake configuration.
-        # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
-        # Use debug flag to show all exeucted tests
-        run: ctest --debug -C ${{matrix.build}}
+        env:
+          # This is the "default-Azurite-connection string", we put it here into the environment (so that it can be picked up by the unittest later on)
+          AZURE_BLOB_STORE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+        run: |
+          # What we do here:
+          # - we create a test CZI file (using the CZIcmd executable)
+          # - we start Azurite (in the background)
+          # - we then upload the test CZI file to Azurite as a blob in the container "testcontainer" with name "testblob"
+          # - we then run the unit-tests - the environment variable AZURE_BLOB_STORE_CONNECTION_STRING is used to configure the Azure SDK based stream tests
+          # - finally, we kill the Azurite process
+          #
+          # find the CZIcmd executable (we just built it)
+          czicmd="$(find . \( -name CZIcmd -o -name CZIcmd.exe \) -print0 | xargs -0 realpath)"
+          mkdir -p azurite
+          cd azurite
+          # now use the CZIcmd executable to create a test CZI file
+          "$czicmd" --command CreateCZI --createbounds "C0:2T0:2" --generatorpixeltype Gray8 --compressionopts "zstd1:ExplicitLevel=2;PreProcess=HiLoByteUnpack" --createsubblocksize "1024x1024" -o test --bitmapgenerator default
+          # start Azurite in the background
+          azurite --inMemoryPersistence --silent &
+          # create a blob container "testcontainer"
+          az storage container create --name testcontainer  --connection-string "$AZURE_BLOB_STORE_CONNECTION_STRING"
+          # upload the test CZI file to the container
+          az storage blob upload --container-name testcontainer --file "./test.czi" --name testblob --connection-string "$AZURE_BLOB_STORE_CONNECTION_STRING"
+          cd ..
+          #"$czicmd" --command PrintInformation --source-stream-class azure_blob_inputstream --source 'account=libczirwtestdata;containername=testcontainer;blobname=testblob;connectionstring=DefaultEndpointsProtocol\=http\;AccountName\=devstoreaccount1\;AccountKey\=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw\=\=\;BlobEndpoint\=http://127.0.0.1:10000/devstoreaccount1\;'
+          #    --propbag-source-stream-creation '{"AzureBlob_AuthenticationMode":"ConnectionString"}'
+          #
+          # Execute tests defined by the CMake configuration.
+          # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
+          # Use debug flag to show all executed tests
+          ctest --debug -C ${{matrix.build}}
+          # note that we leave the Azurite process running, as we want to use it with the code-coverage step as well
+
+      - name: Upload CZICmd as artifact (Windows)
+        working-directory: ${{github.workspace}}/build
+        if: ${{ (matrix.OS == 'windows-latest') && ( matrix.build == 'Release') }}
+        shell: bash
+        run: |
+          mkdir release
+          name="CZICmd-windows-x64-$(git describe --always)"
+          mkdir "release/${name}"
+          cp Src/CZICmd/Release/CZIcmd.exe "release/${name}/"
+          echo "artifactName=${name}" >> "$GITHUB_ENV"
+          echo "artifactPath=${{github.workspace}}/build/release/${name}" >> "$GITHUB_ENV"
+
+      - name: Upload CZICmd as artifact (Linux)
+        working-directory: ${{github.workspace}}/build
+        if: ${{ (matrix.OS == 'ubuntu-latest') && ( matrix.build == 'Release') }}
+        shell: bash
+        run: |
+          mkdir release
+          name="CZICmd-linux-x64-$(git describe --always)"
+          mkdir "release/${name}"
+          cp Src/CZICmd/CZIcmd "release/${name}/"
+          echo "artifactName=${name}" >> "$GITHUB_ENV"
+          echo "artifactPath=${{github.workspace}}/build/release/${name}" >> "$GITHUB_ENV"
+
+      - name: Upload artifacts
+        if: ${{ ( (matrix.OS == 'windows-latest') || (matrix.OS == 'ubuntu-latest') ) && (matrix.build == 'Release') }}
+        uses: actions/upload-artifact@v4
+        with:
+          path: ${{ env.artifactPath }}/
+          name: ${{ env.artifactName }}
 
       # Coverage collection based on https://about.codecov.io/blog/how-to-set-up-codecov-with-c-plus-plus-and-github-actions/
       - name: Prepare Coverage
-        if: ${{ (matrix.OS == 'windows-latest') && ( matrix.build == 'Debug') }}
+        if: ${{ (matrix.OS == 'windows-latest') && (matrix.build == 'Debug') }}
         run: |
           choco install OpenCppCoverage -y --no-progress
           echo "C:\Program Files\OpenCppCoverage" >> "$env:GITHUB_PATH"
 
       - name: Get Coverage
-        if: ${{ (matrix.OS == 'windows-latest') && ( matrix.build == 'Debug') }}
+        if: ${{ (matrix.OS == 'windows-latest') && (matrix.build == 'Debug') }}
         working-directory: ${{github.workspace}}/build/Src/libCZI_UnitTests/${{matrix.build}}
         shell: cmd
         run: OpenCppCoverage.exe --export_type cobertura:${{github.workspace}}\coverage.xml --config_file "${{github.workspace}}\opencppcoverage.txt" -- libCZI_UnitTests.exe
 
       - name: Upload Coverage
         uses: codecov/codecov-action@v4
-        if: ${{ (matrix.OS == 'windows-latest') && ( matrix.build == 'Debug') }}
+        if: ${{ (matrix.OS == 'windows-latest') && (matrix.build == 'Debug') }}
         with:
           files: ./coverage.xml
           fail_ci_if_error: true

.github/workflows/mega-linter.yml

@@ -8,6 +8,7 @@ on:
     branches: ["main"]
   pull_request:
     branches: ["main"]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}
@@ -22,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # MegaLinter
       - name: MegaLinter
@@ -37,7 +38,7 @@ jobs:
 
       # Upload MegaLinter artifacts
       - name: Archive production artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: MegaLinter reports
           path: |

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.15)
 cmake_policy(SET CMP0091 NEW) # enable new "MSVC runtime library selection" (https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html)
 
 project(libCZI 
-      VERSION 0.61.2
+      VERSION 0.62.0
       HOMEPAGE_URL "https://github.com/ZEISS/libczi"
       DESCRIPTION "libCZI is an Open Source Cross-Platform C++ library to read and write CZI")
 
@@ -85,6 +85,11 @@ option(LIBCZI_BUILD_CURL_BASED_STREAM "include curl-based http-/https-stream obj
 # during the CMake-run (and build and use this one).
 option(LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL "Prefer a libcurl package present on the system" OFF)
 
+# This option controls whether to build the Azure-SDK-based-reader object. The Azure-SDK must be externally
+# available (and accessible to CMake's find_package()-command). If using vcpkg, the packages "azure-storage-blobs-cpp"
+# and "azure-identity-cpp" need to be available.
+option(LIBCZI_BUILD_AZURESDK_BASED_STREAM "include AzureSDK-based stream object for accessing Azure-Blob-Store" OFF)
+
 # This option allows to exclude the unit-tests from the build. The unit-tests are using the
 # Google-Test-framework which is downloaded from GitHub during the CMake-run.
 option(LIBCZI_BUILD_UNITTESTS "Build the gTest-based unit-tests" ON)

Src/CMakeLists.txt

@@ -63,6 +63,15 @@ if (LIBCZI_BUILD_CURL_BASED_STREAM)
   endif(LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL)
 endif(LIBCZI_BUILD_CURL_BASED_STREAM)
 
+if(LIBCZI_BUILD_AZURESDK_BASED_STREAM)
+  # -> https://github.com/Azure/azure-sdk-for-cpp#azure-sdk-for-c
+  # -> https://learn.microsoft.com/en-us/azure/storage/blobs/quickstart-blobs-c-plus-plus?tabs=managed-identity%2Croles-azure-portal
+  find_package(azure-identity-cpp CONFIG REQUIRED)
+  find_package(azure-storage-blobs-cpp CONFIG REQUIRED)
+  set(LIBCZI_AZURESDK_VERSION_STRING "core:${azure-core-cpp_VERSION} identity:${azure-identity-cpp_VERSION} storage-blobs:${azure-storage-blobs-cpp_VERSION}")
+  message(STATUS "AZURE-SDK available, version-info: ${LIBCZI_AZURESDK_VERSION_STRING}")
+endif()
+
 add_subdirectory(libCZI)
 
 if (LIBCZI_BUILD_CZICMD)

Src/CZICmd/CMakeLists.txt

@@ -102,7 +102,7 @@ set (CZICMDSRCFILES
 
 add_executable(CZIcmd ${CZICMDSRCFILES})
 
-set_target_properties(CZIcmd PROPERTIES CXX_STANDARD 11)
+set_target_properties(CZIcmd PROPERTIES CXX_STANDARD 14)
 target_compile_definitions(CZIcmd PRIVATE _LIBCZISTATICLIB)
 
 target_link_libraries(CZIcmd PRIVATE ${ZLIB_LIBRARIES}  ${PNG_LIBRARIES} CLI11::CLI11 libCZIStatic)

Src/Doxyfile

@@ -966,6 +966,7 @@ INPUT                  = ./libCZI/libCZI.h \
                          ./libCZI/Doc/using_libCZI.markdown \
                          ./libCZI/Doc/accessors.markdown \
                          ./libCZI/Doc/multichannelcomposition.markdown \
+                         ./libCZI/Doc/stream_objects.markdown \
                          ./libCZI/Doc/CZICmd_usage.markdown \
                          ./libCZI/Doc/write_czi.markdown \
                          ./libCZI/Doc/Todos.markdown \

Src/libCZI/CMakeLists.txt

@@ -116,6 +116,8 @@ set(LIBCZISRCFILES
             StreamsLib/simplefileinputstream.h
             StreamsLib/preadfileinputstream.cpp
             StreamsLib/preadfileinputstream.h
+            StreamsLib/azureblobinputstream.h
+            StreamsLib/azureblobinputstream.cpp
             subblock_cache.h
             subblock_cache.cpp
 )
@@ -226,6 +228,15 @@ else()
   set(libCZI_libcurl_available 0)
 endif()
 
+if (LIBCZI_BUILD_AZURESDK_BASED_STREAM)
+  set(libCZI_AzureStorage_available 1)
+  set(libCZI_AzureStorage_SDK_Version_Info "${LIBCZI_AZURESDK_VERSION_STRING}")
+else()
+  set(libCZI_AzureStorage_available 0)
+  set(libCZI_AzureStorage_SDK_Version_Info "not available")
+endif()
+
+
 if (LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_ZSTD)
   find_package(zstd CONFIG REQUIRED)
 else()
@@ -265,7 +276,7 @@ set(libCZIPublicHeaders "ImportExport.h" "libCZI.h" "libCZI_Compositor.h" "libCZ
 if (LIBCZI_BUILD_DYNLIB)
   add_library(libCZI SHARED ${LIBCZISRCFILES} $<TARGET_OBJECTS:JxrDecodeStatic>)
 
-  set_target_properties(libCZI PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED YES) # https://crascit.com/2015/03/28/enabling-cxx11-in-cmake/
+  set_target_properties(libCZI PROPERTIES CXX_STANDARD 14 CXX_STANDARD_REQUIRED YES) # https://crascit.com/2015/03/28/enabling-cxx11-in-cmake/
   SET_TARGET_PROPERTIES (libCZI PROPERTIES DEFINE_SYMBOL  "LIBCZI_EXPORTS" )
   set_target_properties(libCZI PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION 1)
   # add the binary tree to the search path for include files so that we will find libCZI_Config.h
@@ -282,6 +293,9 @@ if (LIBCZI_BUILD_DYNLIB)
   if (LIBCZI_BUILD_CURL_BASED_STREAM)
     target_link_libraries(libCZI PRIVATE CURL::libcurl)
   endif()
+  if (LIBCZI_BUILD_AZURESDK_BASED_STREAM)
+    target_link_libraries(libCZI PRIVATE Azure::azure-identity Azure::azure-storage-blobs)
+  endif()
   if (LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_EIGEN3)
    target_link_libraries(libCZI PRIVATE Eigen3::Eigen)
   else()
@@ -297,7 +311,7 @@ endif(LIBCZI_BUILD_DYNLIB)
 #
 # Notes: -we use JxrDecode as an "object-library" in order have it "embedded" into libCZI.a
 add_library(libCZIStatic STATIC ${LIBCZISRCFILES} $<TARGET_OBJECTS:JxrDecodeStatic>)
-set_target_properties(libCZIStatic PROPERTIES CXX_STANDARD 11)
+set_target_properties(libCZIStatic PROPERTIES CXX_STANDARD 14)
 target_compile_definitions(libCZIStatic PRIVATE _LIBCZISTATICLIB)
 set_target_properties(libCZIStatic PROPERTIES VERSION ${PROJECT_VERSION})
 # add the binary tree to the search path for include files so that we will find libCZI_Config.h
@@ -315,6 +329,9 @@ endif()
 if (LIBCZI_BUILD_CURL_BASED_STREAM)
   target_link_libraries(libCZIStatic PRIVATE CURL::libcurl)
 endif()
+if (LIBCZI_BUILD_AZURESDK_BASED_STREAM)
+  target_link_libraries(libCZIStatic PRIVATE Azure::azure-identity Azure::azure-storage-blobs)
+endif()
 
 if (LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_EIGEN3)
   target_link_libraries(libCZIStatic PRIVATE Eigen3::Eigen)

Src/libCZI/CziParse.cpp

@@ -794,21 +794,21 @@ using namespace libCZI;
     return (c == 'Y') ? true : false;
 }
 
-/*static*/void CCZIParse::ThrowNotEnoughDataRead(std::uint64_t offset, std::uint64_t bytesRequested, std::uint64_t bytesActuallyRead)
+[[noreturn]] /*static*/void CCZIParse::ThrowNotEnoughDataRead(std::uint64_t offset, std::uint64_t bytesRequested, std::uint64_t bytesActuallyRead)
 {
     stringstream ss;
     ss << "Not enough data read at offset " << offset << " -> requested: " << bytesRequested << " bytes, actually got " << bytesActuallyRead << " bytes.";
     throw LibCZICZIParseException(ss.str().c_str(), LibCZICZIParseException::ErrorCode::NotEnoughData);
 }
 
-/*static*/void CCZIParse::ThrowIllegalData(std::uint64_t offset, const char* sz)
+[[noreturn]] /*static*/void CCZIParse::ThrowIllegalData(std::uint64_t offset, const char* sz)
 {
     stringstream ss;
     ss << "Illegal data detected at offset " << offset << " -> " << sz;
     throw LibCZICZIParseException(ss.str().c_str(), LibCZICZIParseException::ErrorCode::CorruptedData);
 }
 
-/*static*/void CCZIParse::ThrowIllegalData(const char* sz)
+[[noreturn]] /*static*/void CCZIParse::ThrowIllegalData(const char* sz)
 {
     stringstream ss;
     ss << "Illegal data detected -> " << sz;

Src/libCZI/CziParse.h

@@ -210,9 +210,9 @@ class CCZIParse
     static bool IsYDimension(const char* ptr, size_t size);
     static char ToUpperCase(char c);
 
-    static void ThrowNotEnoughDataRead(std::uint64_t offset, std::uint64_t bytesRequested, std::uint64_t bytesActuallyRead);
-    static void ThrowIllegalData(std::uint64_t offset, const char* sz);
-    static void ThrowIllegalData(const char* sz);
+    [[noreturn]] static void ThrowNotEnoughDataRead(std::uint64_t offset, std::uint64_t bytesRequested, std::uint64_t bytesActuallyRead);
+    [[noreturn]] static void ThrowIllegalData(std::uint64_t offset, const char* sz);
+    [[noreturn]] static void ThrowIllegalData(const char* sz);
 
     static bool CheckAttachmentSchemaType(const char* p, size_t cnt);
 };

Src/libCZI/Doc/building_libCZI.markdown

@@ -47,6 +47,7 @@ LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_EIGEN3 | Whether to use an existing Eigen3-l
 LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_ZSTD   | Whether to use an existing zstd-library on the system (included via find_package). If this is OFF, then a copy of zstd is downloaded as part of the build. Default is **OFF**.
 LIBCZI_BUILD_CURL_BASED_STREAM             | Whether a curl-based stream object should be built (and be available in the stream factory). Default is **OFF**.
 LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL| Whether to use an existing libcurl-library on the system (included via find_package). If this is OFF, then a copy of libcurl is downloaded as part of the build. Default is **OFF**.
+LIBCZI_BUILD_AZURESDK_BASED_STREAM         | Whether the Azure-SDK-based stream object should be built (and be available in the stream factory). Default is **OFF**.
 
 If building CZICmd is desired, then running CMake with this command line will enable building CZICmd: