Skip to content

Commit

Permalink
Add intermediate steps and more logging to TaskTracker. (#1640)
Browse files Browse the repository at this point in the history
* Log how long tasks took to complete.

* Add intermediate steps to octree loading.

* Better formatting for intermediate steps.
  • Loading branch information
ThatRedox authored Oct 4, 2023
1 parent 862504f commit f109a51
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 4 deletions.
6 changes: 4 additions & 2 deletions chunky/src/java/se/llbit/chunky/renderer/scene/Scene.java
Original file line number Diff line number Diff line change
Expand Up @@ -2242,17 +2242,19 @@ private synchronized boolean loadOctree(SceneIOProvider context, TaskTracker tas
try {
long fileTimestamp = context.fileTimestamp(fileName);
OctreeFileFormat.OctreeData data;
Consumer<String> stepConsumer = step -> task.update("(1/3) Loading octree (" + step + ")");

try (DataInputStream in = new DataInputStream(new FastBufferedInputStream(new GZIPInputStream(new PositionalInputStream(context.getSceneFileInputStream(fileName), pos -> {
task.updateInterval((int) (pos * progressScale), 1);
}))))) {
data = OctreeFileFormat.load(in, octreeImplementation, this.biomeStructureImplementation);
data = OctreeFileFormat.load(in, octreeImplementation, this.biomeStructureImplementation, stepConsumer);
} catch (PackedOctree.OctreeTooBigException e) {
// Octree too big, reload file and force loading as NodeBasedOctree
Log.warn("Octree was too big when loading dump, reloading with old (slower and bigger) implementation.");
DataInputStream inRetry = new DataInputStream(new FastBufferedInputStream(new GZIPInputStream(new PositionalInputStream(context.getSceneFileInputStream(fileName), pos -> {
task.updateInterval((int) (pos * progressScale), 1);
}))));
data = OctreeFileFormat.load(inRetry, "NODE", this.biomeStructureImplementation);
data = OctreeFileFormat.load(inRetry, "NODE", this.biomeStructureImplementation, stepConsumer);
}

worldOctree = data.worldTree;
Expand Down
16 changes: 14 additions & 2 deletions chunky/src/java/se/llbit/chunky/resources/OctreeFileFormat.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.function.Consumer;

import se.llbit.chunky.block.Block;
import se.llbit.chunky.block.minecraft.Lava;
import se.llbit.chunky.block.minecraft.Water;
Expand All @@ -47,27 +49,37 @@ public class OctreeFileFormat {
* @param in input stream for the file to load the scene from.
* @param octreeImpl The octree implementation to use
* @param legacyBiomeImpl The biome structure implementation to use to load any legacy WorldTextures
* */
public static OctreeData load(DataInputStream in, String octreeImpl, String legacyBiomeImpl) throws IOException {
* @param stepConsumer A consumer that will be called with a string describing the current loading step
*/
public static OctreeData load(DataInputStream in, String octreeImpl, String legacyBiomeImpl, Consumer<String> stepConsumer) throws IOException {
int version = in.readInt();
if (version < MIN_OCTREE_VERSION || version > OCTREE_VERSION) {
throw new IOException(String.format(
"Incompatible octree format: wrong version number (expected %d up to %d, was %d).",
MIN_OCTREE_VERSION, OCTREE_VERSION, version));
}
OctreeData data = new OctreeData();
stepConsumer.accept("block palette");
data.palette = BlockPalette.read(in);
stepConsumer.accept("world octree");
data.worldTree = Octree.load(octreeImpl, version < 5 ? convertDataNodes(data.palette, in) : in);
stepConsumer.accept("water octree");
data.waterTree = Octree.load(octreeImpl, version < 5 ? convertDataNodes(data.palette, in) : in);

if(version >= 7) {
stepConsumer.accept("grass tints");
data.grassColors = loadBiomeStructure(in);
stepConsumer.accept("foliage tints");
data.foliageColors = loadBiomeStructure(in);
stepConsumer.accept("water tints");
data.waterColors = loadBiomeStructure(in);
} else {
stepConsumer.accept("grass tints");
data.grassColors = loadLegacyBiomeStructure(legacyBiomeImpl, in);
stepConsumer.accept("foliage tints");
data.foliageColors = loadLegacyBiomeStructure(legacyBiomeImpl, in);
if (version >= 4) {
stepConsumer.accept("water tints");
data.waterColors = loadLegacyBiomeStructure(legacyBiomeImpl, in);
}
}
Expand Down
10 changes: 10 additions & 0 deletions chunky/src/java/se/llbit/util/TaskTracker.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
*/
package se.llbit.util;

import se.llbit.log.Log;

/**
* A task tracker is used to update a progress listener with current task progress.
* The task tracker has a stack of tasks. When a new task is created the previous
Expand Down Expand Up @@ -89,12 +91,20 @@ public Task(TaskTracker tracker, Task previous, String taskName, int size) {
@Override public void close() {
tracker.currentTask = previous;
previous.update();
Log.infof("Task %s: %d in %.3f seconds", taskName, done,
(System.currentTimeMillis() - startTime) / 1000.0);
}

protected void update() {
tracker.updateProgress(taskName, target, done, eta);
}

/** Change the task name. */
public void update(String task) {
this.taskName = task;
update();
}

/** Set the current progress. */
public void update(int done) {
this.done = done;
Expand Down

0 comments on commit f109a51

Please sign in to comment.