diff --git a/README.md b/README.md index 6b46c87..3025ae6 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ This will produce a pyramid with JPG (lossy) tiles. ## 🚨 Memory overflow issues 🚨 ## +NOTE: Starting from the version 1.1.5, if you omit the icr parameter, the tool will try to "safely" guess it based on the input image dimensions and the heap size. + By default PyramidIO is trying to read and cache the entire image into the memory to achieve the best possible performance. In case of large images it might cause memory overflow issues. There are two types of them: `java.lang.OutOfMemoryError: Java heap space` and `java.lang.RuntimeException: Cannot cache region java.awt.Rectangle` Both are usually fixable by using `-icr` parameter: ``` diff --git a/generic-archiver/pom.xml b/generic-archiver/pom.xml index d13d2d2..41f1f7b 100644 --- a/generic-archiver/pom.xml +++ b/generic-archiver/pom.xml @@ -4,7 +4,7 @@ gov.nist.isg pyramidio-parent - 1.1.4 + 1.1.5 ${project.groupId}:${project.artifactId} Generic library to write archives folders. diff --git a/hdfs-archiver/pom.xml b/hdfs-archiver/pom.xml index 4c6a8f3..d1d7521 100644 --- a/hdfs-archiver/pom.xml +++ b/hdfs-archiver/pom.xml @@ -4,7 +4,7 @@ gov.nist.isg pyramidio-parent - 1.1.4 + 1.1.5 ${project.groupId}:${project.artifactId} Library to read and write archives on HDFS. diff --git a/pom.xml b/pom.xml index 894e16f..bdb08f0 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gov.nist.isg pyramidio-parent - 1.1.4 + 1.1.5 pom ${project.groupId}:${project.artifactId} Parent project of pyramidio, a Java tool to read and write image pyramids. Bio-Formats enhanced version. diff --git a/pyramidio-cli/pom.xml b/pyramidio-cli/pom.xml index f7e056d..7f1bc75 100644 --- a/pyramidio-cli/pom.xml +++ b/pyramidio-cli/pom.xml @@ -4,7 +4,7 @@ gov.nist.isg pyramidio-parent - 1.1.4 + 1.1.5 ${project.groupId}:${project.artifactId} Command line interface to build image pyramids. diff --git a/pyramidio-cli/src/main/java/gov/nist/isg/pyramidio/cli/Main.java b/pyramidio-cli/src/main/java/gov/nist/isg/pyramidio/cli/Main.java index b81e0ac..27efc66 100644 --- a/pyramidio-cli/src/main/java/gov/nist/isg/pyramidio/cli/Main.java +++ b/pyramidio-cli/src/main/java/gov/nist/isg/pyramidio/cli/Main.java @@ -14,9 +14,12 @@ import gov.nist.isg.archiver.FilesArchiver; //@darwinjob import gov.nist.isg.pyramidio.DirectImageReader; import gov.nist.isg.pyramidio.ScalablePyramidBuilder; +import loci.formats.FormatException; +import loci.formats.ImageReader; import no.uio.nesys.pyramidio.BioFormatsImageReader; import java.io.File; +import java.io.IOException; import java.util.logging.Logger; import org.apache.commons.cli.CommandLine; @@ -126,7 +129,7 @@ public static void main(String[] args) { = (Number) commandLine.getParsedOptionValue( inputCacheRatioOption.getOpt()); float cachePercentage = inputCacheRatioNumber == null - ? 1 + ? getCalculatedPercentage(inputFile) : inputCacheRatioNumber.floatValue(); ScalablePyramidBuilder spb = new ScalablePyramidBuilder( @@ -164,7 +167,61 @@ public static void main(String[] args) { } - private static void printHelp(Options options) { + private static float getCalculatedPercentage(File inputFile) { + + try { + ImageReader imageReader = new ImageReader(); + imageReader.setId(inputFile.getPath()); + + long width = imageReader.getSizeX(); + long height = imageReader.getSizeY(); + long bitsPerPixel = imageReader.getBitsPerPixel(); + long channelCount = imageReader.getRGBChannelCount(); + + logger.info("Input image width: " + width + ", height: " + height + ", channel count: " + channelCount + + ", bits per pixel: " + bitsPerPixel); + if (channelCount > 3) + logger.warning( + "Channel count > 3. Is alpha channel present? The output pyramid might be corrupted. See https://github.com/darwinjob/pyramidio-bioformats/issues/1"); + + long imageSize = width * height * bitsPerPixel * channelCount / 8; // bytes + logger.info("Uncompressed image size: " + imageSize / (1024 * 1024) + "MB"); + + logger.info("Number of processors: " + Runtime.getRuntime().availableProcessors()); + + // Get current size of heap in bytes + long heapSize = Runtime.getRuntime().totalMemory(); + // Get maximum size of heap in bytes. The heap cannot grow beyond this size. Any + // attempt will result in an OutOfMemoryException. + long heapMaxSize = Runtime.getRuntime().maxMemory(); + // Get amount of free memory within the heap in bytes. This size will increase + // after garbage collection and decrease as new objects are created. + long heapFreeSize = Runtime.getRuntime().freeMemory(); + + logger.info("Total memory: " + heapSize / (1024 * 1024) + "MB, max memory: " + heapMaxSize / (1024 * 1024) + + "MB, free memory: " + heapFreeSize / (1024 * 1024) + "MB"); + + // TODO ATTN.!: Assuming taking this amount of the heap is safe. Wild guess. + float icr = (heapMaxSize / 5f) / imageSize; + logger.info("Calculated icr: " + icr); + + if (icr > 1) + icr = 1; + + logger.info("Actual icr: " + icr); + + imageReader.close(); + return icr; + } catch (FormatException | IOException e) { + logger.info( + "Failed to calculate icr (Input Cache Ratio). Setting icr to 1 (cache entire image). The exception:\n" + + e); + return 1; + } + + } + + private static void printHelp(Options options) { new HelpFormatter().printHelp("pyramidio", options); } } diff --git a/pyramidio/pom.xml b/pyramidio/pom.xml index 80d1b6e..8df751f 100644 --- a/pyramidio/pom.xml +++ b/pyramidio/pom.xml @@ -6,7 +6,7 @@ gov.nist.isg pyramidio-parent - 1.1.4 + 1.1.5 ${project.groupId}:${project.artifactId} Library to read and write image pyramids. diff --git a/s3-archiver/pom.xml b/s3-archiver/pom.xml index 377ca2b..454c091 100644 --- a/s3-archiver/pom.xml +++ b/s3-archiver/pom.xml @@ -4,7 +4,7 @@ gov.nist.isg pyramidio-parent - 1.1.4 + 1.1.5 ${project.groupId}:${project.artifactId} Library to read and write archives on Amazon S3. diff --git a/tar-archiver/pom.xml b/tar-archiver/pom.xml index 3e84541..a946366 100644 --- a/tar-archiver/pom.xml +++ b/tar-archiver/pom.xml @@ -4,7 +4,7 @@ gov.nist.isg pyramidio-parent - 1.1.4 + 1.1.5 ${project.groupId}:${project.artifactId} Library to read and write archives in tar files.