VRTProcessedDataset: reduce block size when amount of RAM for temp bu…

…ffers would be too large
OSGeo · Jan 3, 2025 · 77f0259 · 77f0259
1 parent 1c49426
commit 77f0259
Showing 1 changed file with 56 additions and 0 deletions.
diff --git a/frmts/vrt/vrtprocesseddataset.cpp b/frmts/vrt/vrtprocesseddataset.cpp
@@ -301,12 +301,19 @@ CPLErr VRTProcessedDataset::Init(const CPLXMLNode *psTree,
 
     const auto poSrcFirstBand = m_poSrcDS->GetRasterBand(1);
     poSrcFirstBand->GetBlockSize(&m_nBlockXSize, &m_nBlockYSize);
+    bool bUserBlockSize = false;
     if (const char *pszBlockXSize =
             CPLGetXMLValue(psTree, "BlockXSize", nullptr))
+    {
+        bUserBlockSize = true;
         m_nBlockXSize = atoi(pszBlockXSize);
+    }
     if (const char *pszBlockYSize =
             CPLGetXMLValue(psTree, "BlockYSize", nullptr))
+    {
+        bUserBlockSize = true;
         m_nBlockYSize = atoi(pszBlockYSize);
+    }
 
     // Initialize all the general VRT stuff.
     if (VRTDataset::XMLInit(psTree, pszVRTPathIn) != CE_None)
@@ -521,6 +528,55 @@ CPLErr VRTProcessedDataset::Init(const CPLXMLNode *psTree,
         return CE_Failure;
     }
 
+    // Use only up to 40% of RAM to acquire source bands and generate the output
+    // buffer.
+    const auto nMaxMemAllowed = CPLGetUsablePhysicalRAM() / 10 * 4;
+    if (!bUserBlockSize && nMaxMemAllowed > 0)
+    {
+        bool bBlockSizeModified = false;
+        const int nFirstBandCount = m_aoSteps.front().nInBands;
+        CPLAssert(nFirstBandCount == m_poSrcDS->GetRasterCount());
+        const GDALDataType eFirstDT = m_aoSteps.front().eInDT;
+        const int nFirstDTSize = GDALGetDataTypeSizeBytes(eFirstDT);
+
+        int nLargestDTSizeTimesBand = 0;
+        for (const auto &oStep : m_aoSteps)
+        {
+            const int nOutDTSizeTimesBand =
+                GDALGetDataTypeSizeBytes(oStep.eOutDT) * oStep.nOutBands;
+            nLargestDTSizeTimesBand =
+                std::max(nLargestDTSizeTimesBand, nOutDTSizeTimesBand);
+        }
+
+        const int nPerPixelSize =
+            nFirstBandCount * nFirstDTSize + nLargestDTSizeTimesBand;
+
+        while ((m_nBlockXSize >= 2 || m_nBlockYSize >= 2) &&
+               static_cast<GIntBig>(m_nBlockXSize) * m_nBlockYSize *
+                       nPerPixelSize >
+                   nMaxMemAllowed)
+        {
+            if ((m_nBlockXSize == nRasterXSize ||
+                 m_nBlockYSize >= m_nBlockXSize) &&
+                m_nBlockYSize >= 2)
+            {
+                m_nBlockYSize /= 2;
+            }
+            else
+            {
+                m_nBlockXSize /= 2;
+            }
+            bBlockSizeModified = true;
+        }
+        if (bBlockSizeModified)
+        {
+            CPLDebug("VRT",
+                     "Reducing block size to %d x %d to avoid consuming too "
+                     "much RAM",
+                     m_nBlockXSize, m_nBlockYSize);
+        }
+    }
+
     if (m_outputBandCountProvenance == ValueProvenance::FROM_LAST_STEP)
     {
         nOutputBandCount = nCurrentBandCount;