From d9787d9b0cef9c187aeb0479f56dac15c1852de4 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 17 Sep 2023 11:27:09 -0700 Subject: [PATCH 1/7] Add Interner interface and WeakInterner implementation. --- .../java/se/llbit/util/interner/Interner.java | 29 ++++ .../se/llbit/util/interner/WeakInterner.java | 93 +++++++++++++ .../test/se/llbit/util/WeakInternerTest.java | 127 ++++++++++++++++++ 3 files changed, 249 insertions(+) create mode 100644 chunky/src/java/se/llbit/util/interner/Interner.java create mode 100644 chunky/src/java/se/llbit/util/interner/WeakInterner.java create mode 100644 chunky/src/test/se/llbit/util/WeakInternerTest.java diff --git a/chunky/src/java/se/llbit/util/interner/Interner.java b/chunky/src/java/se/llbit/util/interner/Interner.java new file mode 100644 index 0000000000..8219af2864 --- /dev/null +++ b/chunky/src/java/se/llbit/util/interner/Interner.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Chunky contributors + * + * This file is part of Chunky. + * + * Chunky is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chunky is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with Chunky. If not, see . + */ + +package se.llbit.util.interner; + +/** + * This deduplicates objects similar to `String.intern()`, but for arbitrary objects. + */ +public interface Interner { + /** + * Intern the given object, returning an existing object if one exists. + */ + T intern(T sample); +} diff --git a/chunky/src/java/se/llbit/util/interner/WeakInterner.java b/chunky/src/java/se/llbit/util/interner/WeakInterner.java new file mode 100644 index 0000000000..9ed0b406eb --- /dev/null +++ b/chunky/src/java/se/llbit/util/interner/WeakInterner.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2023 Chunky contributors + * + * This file is part of Chunky. + * + * Chunky is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chunky is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with Chunky. If not, see . + */ + +package se.llbit.util.interner; + +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; +import java.util.HashMap; + +/** + * A simple interner that keeps weak references to interned objects. + */ +public class WeakInterner implements Interner { + protected static class HashedWeakReference extends WeakReference { + private final int hash; + + public HashedWeakReference(T referent, ReferenceQueue q) { + super(referent, q); + this.hash = referent.hashCode(); + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof HashedWeakReference)) { + return false; + } + HashedWeakReference other = (HashedWeakReference) obj; + + Object self = this.get(); + if (self == null) { + return false; + } + + return self.equals(other.get()); + } + } + + protected final HashMap, HashedWeakReference> pool = new HashMap<>(); + protected final ReferenceQueue queue = new ReferenceQueue<>(); + + @Override + public T intern(T sample) { + compact(); + + HashedWeakReference ref = new HashedWeakReference<>(sample, queue); + HashedWeakReference existing = pool.get(ref); + + T obj; + if (existing != null && (obj = existing.get()) != null) { + // Existing object exists and is alive + // Don't enqueue our new ref + ref.clear(); + return obj; + } else { + // Existing object is dead or doesn't exist + // Enqueue our new ref + pool.put(ref, ref); + return sample; + } + } + + /** + * Compact this interner by cleaning up dead references. + */ + public void compact() { + HashedWeakReference ref; + do { + //noinspection unchecked - only HashedWeakReference objects are added to the queue + ref = (HashedWeakReference) queue.poll(); + pool.remove(ref); + } while (ref != null); + } +} diff --git a/chunky/src/test/se/llbit/util/WeakInternerTest.java b/chunky/src/test/se/llbit/util/WeakInternerTest.java new file mode 100644 index 0000000000..df8237188c --- /dev/null +++ b/chunky/src/test/se/llbit/util/WeakInternerTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 Chunky contributors + * + * This file is part of Chunky. + * + * Chunky is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chunky is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with Chunky. If not, see . + */ + +package se.llbit.util; + +import org.junit.Assume; +import org.junit.Ignore; +import org.junit.Test; +import se.llbit.util.interner.Interner; +import se.llbit.util.interner.WeakInterner; + +import java.lang.ref.WeakReference; +import java.util.Objects; +import java.util.Random; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class WeakInternerTest { + private static class TestInternable { + public final long value; + + public TestInternable(long value) { + this.value = value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TestInternable)) return false; + TestInternable that = (TestInternable) o; + return value == that.value; + } + + @Override + public int hashCode() { + return Objects.hash(value); + } + } + + @Test + public void testSmoke() { + Random rand = new Random(0); + Interner interner = new WeakInterner<>(); + + TestInternable test11 = new TestInternable(rand.nextLong()); + TestInternable test12 = new TestInternable(rand.nextLong()); + TestInternable test13 = new TestInternable(rand.nextLong()); + TestInternable test21 = new TestInternable(test11.value); + TestInternable test22 = new TestInternable(test12.value); + TestInternable test23 = new TestInternable(test13.value); + + // Test that we get the same object back when interning the object. + assertSame(interner.intern(test11), test11); + assertSame(interner.intern(test12), test12); + assertSame(interner.intern(test13), test13); + + // Test that we get back the interned object when interning the copy + assertSame(interner.intern(test21), test11); + assertSame(interner.intern(test22), test12); + assertSame(interner.intern(test23), test13); + } + + @Test + public void testWeakness() { + Random rand = new Random(0); + Interner interner = new WeakInterner<>(); + + TestInternable test11 = new TestInternable(rand.nextLong()); + TestInternable test12 = new TestInternable(rand.nextLong()); + TestInternable test13 = new TestInternable(rand.nextLong()); + TestInternable test21 = new TestInternable(test11.value); + TestInternable test22 = new TestInternable(test12.value); + TestInternable test23 = new TestInternable(test13.value); + + // Test that we get the same object back when interning the object. + assertSame(interner.intern(test11), test11); + assertSame(interner.intern(test12), test12); + assertSame(interner.intern(test13), test13); + + // Test that we get back the interned object when interning the copy + assertSame(interner.intern(test21), test11); + assertSame(interner.intern(test22), test12); + assertSame(interner.intern(test23), test13); + + // Get references to the interned objects + WeakReference weakTest1 = new WeakReference<>(test11); + WeakReference weakTest2 = new WeakReference<>(test12); + WeakReference weakTest3 = new WeakReference<>(test13); + + // Clear their references + test11 = null; + test12 = null; + test13 = null; + + // Wait until they are garbage collected + for (int i = 0; i < 1000; i++) { + System.gc(); + if (weakTest1.get() == null && weakTest2.get() == null && weakTest3.get() == null) { + break; + } + } + Assume.assumeTrue(weakTest1.get() == null); + Assume.assumeTrue(weakTest2.get() == null); + Assume.assumeTrue(weakTest3.get() == null); + + // Test that we get the copy back when interning the copy + assertSame(interner.intern(test21), test21); + assertSame(interner.intern(test22), test22); + assertSame(interner.intern(test23), test23); + } +} From 53a868a4cb0469819a3eb905368d483297830e69 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 17 Sep 2023 11:36:52 -0700 Subject: [PATCH 2/7] Separate compact and endFinalization. --- .../src/java/se/llbit/chunky/renderer/scene/Scene.java | 10 +++++++--- .../chunky/renderer/scene/biome/BiomeStructure.java | 10 +++++++--- .../renderer/scene/biome/Trivial2dBiomeStructure.java | 4 ---- .../renderer/scene/biome/Trivial3dBiomeStructure.java | 5 ----- .../scene/biome/WorldTexture2dBiomeStructure.java | 2 +- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/Scene.java b/chunky/src/java/se/llbit/chunky/renderer/scene/Scene.java index 53ffa7996d..64cafa4aec 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/Scene.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/Scene.java @@ -1385,14 +1385,18 @@ public synchronized void loadChunks(TaskTracker taskTracker, World world, Collec } task.updateEta(target, done); done += 1; + + grassTexture.compact(); + foliageTexture.compact(); + waterTexture.compact(); } worldOctree.endFinalization(); waterOctree.endFinalization(); - grassTexture.compact(); - foliageTexture.compact(); - waterTexture.compact(); + grassTexture.endFinalization(); + foliageTexture.endFinalization(); + waterTexture.endFinalization(); } entities.loadDataFromOctree(worldOctree, palette, origin); diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java index 1b2cc04ec5..4dabfc8484 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java @@ -84,10 +84,14 @@ static Factory get(@NotNull String key) throws NullPointerException { void store(DataOutputStream out) throws IOException; /** - * This method is called to tell the implementation to shrink its size. (Node-tree optimisation, etc.) - * Called when throughout insertion of new biomes, and on completion + * This method is called to tell the implementation to shrink its size. Called when throughout insertion of new biomes. */ - void compact(); + default void compact() {} + + /** + * This method is called to tell the implementation to finalize. Called on completion of loading. + */ + default void endFinalization() {} /** * @return The registry key this biome format uses. Must be unique diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/Trivial2dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/Trivial2dBiomeStructure.java index 050cc8f81f..a8d7677bde 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/Trivial2dBiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/Trivial2dBiomeStructure.java @@ -114,9 +114,5 @@ public void store(DataOutputStream out) throws IOException { public String biomeFormat() { return ID; } - - @Override - public void compact() { - } } } diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/Trivial3dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/Trivial3dBiomeStructure.java index 6c42fb27b2..c2d575d958 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/Trivial3dBiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/Trivial3dBiomeStructure.java @@ -120,10 +120,5 @@ public void store(DataOutputStream out) throws IOException { public String biomeFormat() { return ID; } - - @Override - public void compact() { - - } } } diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/WorldTexture2dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/WorldTexture2dBiomeStructure.java index ac8072a7cf..ea855ce7ae 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/WorldTexture2dBiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/WorldTexture2dBiomeStructure.java @@ -52,7 +52,7 @@ public void store(DataOutputStream out) throws IOException { } @Override - public void compact() { + public void endFinalization() { texture.compact(); } From 4d40818d73a29f1fca5062df01a0458515522d1b Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 17 Sep 2023 16:49:10 -0700 Subject: [PATCH 3/7] Add StrongInterner. Move and refactor WorldTexture implementation. --- .../renderer/scene/biome/BiomeStructure.java | 5 +- .../biome/WorldTexture2dBiomeStructure.java | 74 --------- .../biome/worldtexture}/ChunkTexture.java | 105 ++++++------ .../WorldTexture2dBiomeStructure.java | 144 +++++++++++++++++ .../se/llbit/chunky/world/WorldTexture.java | 153 ------------------ .../llbit/util/interner/StrongInterner.java | 33 ++++ 6 files changed, 234 insertions(+), 280 deletions(-) delete mode 100644 chunky/src/java/se/llbit/chunky/renderer/scene/biome/WorldTexture2dBiomeStructure.java rename chunky/src/java/se/llbit/chunky/{world => renderer/scene/biome/worldtexture}/ChunkTexture.java (61%) create mode 100644 chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java delete mode 100644 chunky/src/java/se/llbit/chunky/world/WorldTexture.java create mode 100644 chunky/src/java/se/llbit/util/interner/StrongInterner.java diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java index 4dabfc8484..205e4f1e43 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java @@ -1,8 +1,8 @@ package se.llbit.chunky.renderer.scene.biome; import it.unimi.dsi.fastutil.objects.Object2ReferenceOpenHashMap; +import se.llbit.chunky.renderer.scene.biome.worldtexture.WorldTexture2dBiomeStructure; import se.llbit.chunky.world.Chunk; -import se.llbit.chunky.world.WorldTexture; import se.llbit.log.Log; import se.llbit.math.structures.Position2IntStructure; import se.llbit.math.structures.Position2ReferenceStructure; @@ -28,9 +28,6 @@ static void registerDefaults() { } /** - * This is basically a reimplementation of {@link WorldTexture#load} but instead loading into an arbitrary - * BiomeStructure implementation - * * @param impl The implementation to load the legacy implementation into * @param in The serialised legacy data in an input stream * @return The newly constructed {@link BiomeStructure} of the specified implementation diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/WorldTexture2dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/WorldTexture2dBiomeStructure.java deleted file mode 100644 index ea855ce7ae..0000000000 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/WorldTexture2dBiomeStructure.java +++ /dev/null @@ -1,74 +0,0 @@ -package se.llbit.chunky.renderer.scene.biome; - -import se.llbit.chunky.world.WorldTexture; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; - -public class WorldTexture2dBiomeStructure implements BiomeStructure.Factory { - static final String ID = "WORLD_TEXTURE_2D"; - - @Override - public BiomeStructure create() { - return new Impl(new WorldTexture()); - } - - @Override - public BiomeStructure load(DataInputStream in) throws IOException { - return new Impl(WorldTexture.load(in)); - } - - @Override - public boolean is3d() { - return false; - } - - @Override - public String getName() { - return "World texture 2d"; - } - - @Override - public String getDescription() { - return "A 2d biome format that uses de-duplicated bitmaps per chunk."; - } - - @Override - public String getId() { - return ID; - } - - static class Impl implements BiomeStructure { - private final WorldTexture texture; - - public Impl(WorldTexture texture) { - this.texture = texture; - } - - @Override - public void store(DataOutputStream out) throws IOException { - texture.store(out); - } - - @Override - public void endFinalization() { - texture.compact(); - } - - @Override - public String biomeFormat() { - return ID; - } - - @Override - public void set(int x, int y, int z, float[] data) { - texture.set(x, z, data); - } - - @Override - public float[] get(int x, int y, int z) { - return texture.get(x, z); - } - } -} diff --git a/chunky/src/java/se/llbit/chunky/world/ChunkTexture.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java similarity index 61% rename from chunky/src/java/se/llbit/chunky/world/ChunkTexture.java rename to chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java index 6b819d5057..856f77c07c 100644 --- a/chunky/src/java/se/llbit/chunky/world/ChunkTexture.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java @@ -1,4 +1,5 @@ -/* Copyright (c) 2012-2013 Jesper Öqvist +/* + * Copyright (c) 2023 Chunky contributors * * This file is part of Chunky. * @@ -14,8 +15,10 @@ * You should have received a copy of the GNU General Public License * along with Chunky. If not, see . */ -package se.llbit.chunky.world; +package se.llbit.chunky.renderer.scene.biome.worldtexture; + +import se.llbit.chunky.world.Chunk; import se.llbit.math.ColorUtil; import java.io.DataInputStream; @@ -23,14 +26,9 @@ import java.io.IOException; import java.util.Arrays; -/** - * Chunk texture - * - * @author Jesper Öqvist - */ -public class ChunkTexture { - +public class ChunkTexture { protected final byte[] data = new byte[Chunk.X_MAX * Chunk.Z_MAX * 3]; + protected boolean writeable = true; /** * Create new texture @@ -46,36 +44,23 @@ public ChunkTexture(ChunkTexture ct) { } /** - * Set color value at (x, z). - * - * @param frgb RGB color components to set - */ - public void set(int x, int z, float[] frgb) { - int index = (x + z * Chunk.X_MAX) * 3; - data[index] = ColorUtil.RGBComponentFromLinear(frgb[0]); - data[index + 1] = ColorUtil.RGBComponentFromLinear(frgb[1]); - data[index + 2] = ColorUtil.RGBComponentFromLinear(frgb[2]); - } - - /** - * @return RGB color components at (x, z) + * Load a chunk texture from an input stream. */ - public float[] get(int x, int z) { - float[] result = new float[3]; - int index = (x + z * Chunk.X_MAX) * 3; - result[0] = ColorUtil.RGBComponentToLinear(data[index]); - result[1] = ColorUtil.RGBComponentToLinear(data[index + 1]); - result[2] = ColorUtil.RGBComponentToLinear(data[index + 2]); - return result; + public static ChunkTexture load(DataInputStream in) throws IOException { + ChunkTexture tex = new ChunkTexture(); + for (int i = 0; i < Chunk.X_MAX * Chunk.Z_MAX; i++) { + tex.data[i*3 + 0] = ColorUtil.RGBComponentFromLinear(in.readFloat()); + tex.data[i*3 + 1] = ColorUtil.RGBComponentFromLinear(in.readFloat()); + tex.data[i*3 + 2] = ColorUtil.RGBComponentFromLinear(in.readFloat()); + } + return tex; } /** * Write this chunk texture to an output stream. - * - * @throws IOException */ public void store(DataOutputStream out) throws IOException { - for (int i = 0; i < Chunk.X_MAX * Chunk.Z_MAX; ++i) { + for (int i = 0; i < Chunk.X_MAX * Chunk.Z_MAX; i++) { out.writeFloat(ColorUtil.RGBComponentToLinear(data[i*3])); out.writeFloat(ColorUtil.RGBComponentToLinear(data[i*3 + 1])); out.writeFloat(ColorUtil.RGBComponentToLinear(data[i*3 + 2])); @@ -83,31 +68,53 @@ public void store(DataOutputStream out) throws IOException { } /** - * Load a chunk texture from an input stream. - * - * @return The loaded texture - * @throws IOException + * Make this chunk texture read-only. */ - public static ChunkTexture load(DataInputStream in) throws IOException { - ChunkTexture texture = new ChunkTexture(); - for (int i = 0; i < Chunk.X_MAX * Chunk.Z_MAX; ++i) { - texture.data[i*3] = ColorUtil.RGBComponentFromLinear(in.readFloat()); - texture.data[i*3 + 1] = ColorUtil.RGBComponentFromLinear(in.readFloat()); - texture.data[i*3 + 2] = ColorUtil.RGBComponentFromLinear(in.readFloat()); + public void makeReadOnly() { + writeable = false; + } + + /** + * Set the color value at (x, z). + * @param frgb RGB color components to set + * @return The chunk texture with the color value set + */ + public ChunkTexture set(int x, int z, float[] frgb) { + if (writeable) { + int index = (x + z * Chunk.X_MAX) * 3; + data[index + 0] = ColorUtil.RGBComponentFromLinear(frgb[0]); + data[index + 1] = ColorUtil.RGBComponentFromLinear(frgb[1]); + data[index + 2] = ColorUtil.RGBComponentFromLinear(frgb[2]); + return this; + } else { + ChunkTexture tex = new ChunkTexture(this); + return tex.set(x, z, frgb); } - return texture; } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ChunkTexture that = (ChunkTexture) o; - return Arrays.equals(data, that.data); + /** + * Get the color value at (x, z). + * @return RGB color components at (x, z) + */ + public float[] get(int x, int z) { + float[] result = new float[3]; + int index = (x + z * Chunk.X_MAX) * 3; + result[0] = ColorUtil.RGBComponentToLinear(data[index + 0]); + result[1] = ColorUtil.RGBComponentToLinear(data[index + 1]); + result[2] = ColorUtil.RGBComponentToLinear(data[index + 2]); + return result; } @Override public int hashCode() { return Arrays.hashCode(data); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ChunkTexture)) return false; + ChunkTexture that = (ChunkTexture) o; + return Arrays.equals(data, that.data); + } } diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java new file mode 100644 index 0000000000..dab56c889b --- /dev/null +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2023 Chunky contributors + * + * This file is part of Chunky. + * + * Chunky is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chunky is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with Chunky. If not, see . + */ + +package se.llbit.chunky.renderer.scene.biome.worldtexture; + +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import se.llbit.chunky.renderer.scene.biome.BiomeStructure; +import se.llbit.util.interner.Interner; +import se.llbit.util.interner.StrongInterner; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +public class WorldTexture2dBiomeStructure implements BiomeStructure.Factory { + public static final String ID = "WORLD_TEXTURE_2D"; + + @Override + public BiomeStructure create() { + return new Impl(); + } + + @Override + public BiomeStructure load(DataInputStream in) throws IOException { + return Impl.load(in); + } + + @Override + public boolean is3d() { + return false; + } + + @Override + public String getName() { + return "World texture 2d"; + } + + @Override + public String getDescription() { + return "A 2d biome format that uses de-duplicated bitmaps per chunk."; + } + + @Override + public String getId() { + return ID; + } + + static class Impl implements BiomeStructure { + private final Long2ObjectOpenHashMap map = new Long2ObjectOpenHashMap<>(); + + private static long chunkPos(int x, int z) { + return (((long) x) << 32) | ((long) z); + } + + public static Impl load(DataInputStream in) throws IOException { + Impl texture = new Impl(); + Interner interner = new StrongInterner<>(); + + int numTiles = in.readInt(); + for (int i = 0; i < numTiles; i++) { + int x = in.readInt(); + int z = in.readInt(); + ChunkTexture tile = ChunkTexture.load(in); + tile = interner.intern(tile); + texture.map.put(chunkPos(x, z), tile); + } + + return texture; + } + + @Override + public void store(DataOutputStream out) throws IOException { + out.writeInt(map.size()); + for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { + long pos = entry.getLongKey(); + ChunkTexture texture = entry.getValue(); + + out.writeInt((int) (pos >> 32)); + out.writeInt((int) pos); + texture.store(out); + } + } + + @Override + public void endFinalization() { + Interner interner = new StrongInterner<>(); + for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { + ChunkTexture texture = entry.getValue(); + ChunkTexture interned = interner.intern(texture); + + // We did de-duplicate, mark the texture as shared and replace the value + if (interned != texture) { + interned.makeReadOnly(); + entry.setValue(interned); + } + } + } + + @Override + public String biomeFormat() { + return ID; + } + + @Override + public void set(int x, int y, int z, float[] data) { + long cp = chunkPos(x >> 4, z >> 4); + ChunkTexture texture = map.get(cp); + + if (texture == null) { + texture = new ChunkTexture(); + } + + ChunkTexture newTex = texture.set(x & 0xF, z & 0xF, data); + if (newTex != texture) { + map.put(cp, newTex); + } + } + + @Override + public float[] get(int x, int y, int z) { + ChunkTexture texture = map.get(chunkPos(x >> 4, z >> 4)); + if (texture == null) { + return null; + } + return texture.get(x & 0xF, z & 0xF); + } + } +} diff --git a/chunky/src/java/se/llbit/chunky/world/WorldTexture.java b/chunky/src/java/se/llbit/chunky/world/WorldTexture.java deleted file mode 100644 index 5c31b0b347..0000000000 --- a/chunky/src/java/se/llbit/chunky/world/WorldTexture.java +++ /dev/null @@ -1,153 +0,0 @@ -/* Copyright (c) 2012-2014 Jesper Öqvist - * - * This file is part of Chunky. - * - * Chunky is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Chunky is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * along with Chunky. If not, see . - */ -package se.llbit.chunky.world; - -import it.unimi.dsi.fastutil.longs.Long2ObjectMap; -import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.HashMap; - -/** - * World texture. - * - * @author Jesper Öqvist - */ -public class WorldTexture { - - private final Long2ObjectOpenHashMap map = new Long2ObjectOpenHashMap<>(); - - private boolean compacted = false; - - /** - * Timestamp of last serialization. - */ - private long timestamp = 0; - - /** - * Set color at (x, z) - * - * @param frgb RGB color components - * @throws IllegalStateException if this WorldTexture has already been finalized. - */ - public void set(int x, int z, float[] frgb) { - long cp = ((long) x >> 4) << 32 | ((z >> 4) & 0xffffffffL); - ChunkTexture ct = map.get(cp); - - // This chunk texture might be used in other places so we must create a copy - if (compacted) { - ct = new ChunkTexture(ct); - } - - if (ct == null) { - ct = new ChunkTexture(); - map.put(cp, ct); - } - ct.set(x & 0xF, z & 0xF, frgb); - } - - /** - * @return True if this texture contains a RGB color components at (x, z) - */ - public boolean contains(int x, int z) { - long cp = ((long) x >> 4) << 32 | ((z >> 4) & 0xffffffffL); - return map.containsKey(cp); - } - - /** - * @return RGB color components at (x, z) - */ - public float[] get(int x, int z) { - long cp = ((long) x >> 4) << 32 | ((z >> 4) & 0xffffffffL); - ChunkTexture ct = map.get(cp); - if (ct == null) { - return null; - } - return ct.get(x & 0xF, z & 0xF); - } - - /** - * Write the world texture to the output stream - * - * @throws IOException - */ - public void store(DataOutputStream out) throws IOException { - out.writeInt(map.size()); - for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { - long pos = entry.getLongKey(); - ChunkTexture texture = entry.getValue(); - out.writeInt((int) (pos >> 32)); - out.writeInt((int) pos); - texture.store(out); - } - } - - /** - * Load world texture from the input stream - * - * @return Loaded texture - * @throws IOException - */ - public static WorldTexture load(DataInputStream in) throws IOException { - WorldTexture texture = new WorldTexture(); - HashMap textureCache = new HashMap<>(); - int numTiles = in.readInt(); - for (int i = 0; i < numTiles; ++i) { - int x = in.readInt(); - int z = in.readInt(); - ChunkTexture tile = ChunkTexture.load(in); - tile = textureCache.computeIfAbsent(tile, t -> t); - texture.map.put(((long) x) << 32 | (z & 0xffffffffL), tile); - } - return texture; - } - - /** - * Deduplicate this {@code WorldTexture} to save memory. This also makes this read-only. - */ - public void compact() { - if (compacted) return; - compacted = true; - - HashMap textureCache = new HashMap<>(); - for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { - ChunkTexture tile = entry.getValue(); - - if (textureCache.containsKey(tile)) { - entry.setValue(textureCache.get(tile)); - } else { - textureCache.put(tile, tile); - } - } - } - - /** - * @return last serialization timestamp - */ - public long getTimestamp() { - return timestamp; - } - - /** - * Set the serialization timestamp. - */ - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } -} diff --git a/chunky/src/java/se/llbit/util/interner/StrongInterner.java b/chunky/src/java/se/llbit/util/interner/StrongInterner.java new file mode 100644 index 0000000000..3651fec74f --- /dev/null +++ b/chunky/src/java/se/llbit/util/interner/StrongInterner.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 Chunky contributors + * + * This file is part of Chunky. + * + * Chunky is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chunky is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with Chunky. If not, see . + */ + +package se.llbit.util.interner; + +import java.util.HashMap; + +/** + * A simple interner that keeps strong references to interned objects. + */ +public class StrongInterner implements Interner { + protected final HashMap pool = new HashMap<>(); + + @Override + public T intern(T sample) { + return pool.computeIfAbsent(sample, k -> k); + } +} From 21f3ada48ad0581733a88fdf4ab23e0b9911a6a7 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 17 Sep 2023 17:38:32 -0700 Subject: [PATCH 4/7] Add `maybeIntern` to Interner & use it. --- .../biome/worldtexture/ChunkTexture.java | 2 +- .../WorldTexture2dBiomeStructure.java | 12 +++-- .../java/se/llbit/util/interner/Interner.java | 14 +++++- .../llbit/util/interner/StrongInterner.java | 10 ++++ .../se/llbit/util/interner/WeakInterner.java | 4 +- .../test/se/llbit/util/WeakInternerTest.java | 46 +++++++++++++++++-- 6 files changed, 77 insertions(+), 11 deletions(-) diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java index 856f77c07c..9345267294 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java @@ -28,7 +28,7 @@ public class ChunkTexture { protected final byte[] data = new byte[Chunk.X_MAX * Chunk.Z_MAX * 3]; - protected boolean writeable = true; + protected transient boolean writeable = true; /** * Create new texture diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java index dab56c889b..da3ccff576 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java @@ -77,6 +77,13 @@ public static Impl load(DataInputStream in) throws IOException { int x = in.readInt(); int z = in.readInt(); ChunkTexture tile = ChunkTexture.load(in); + + ChunkTexture interned = interner.maybeIntern(tile); + if (interned != null) { + interned.makeReadOnly(); + tile = interned; + } + tile = interner.intern(tile); texture.map.put(chunkPos(x, z), tile); } @@ -101,11 +108,10 @@ public void store(DataOutputStream out) throws IOException { public void endFinalization() { Interner interner = new StrongInterner<>(); for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { - ChunkTexture texture = entry.getValue(); - ChunkTexture interned = interner.intern(texture); + ChunkTexture interned = interner.maybeIntern(entry.getValue()); // We did de-duplicate, mark the texture as shared and replace the value - if (interned != texture) { + if (interned != null) { interned.makeReadOnly(); entry.setValue(interned); } diff --git a/chunky/src/java/se/llbit/util/interner/Interner.java b/chunky/src/java/se/llbit/util/interner/Interner.java index 8219af2864..28f7ca66d0 100644 --- a/chunky/src/java/se/llbit/util/interner/Interner.java +++ b/chunky/src/java/se/llbit/util/interner/Interner.java @@ -22,8 +22,20 @@ * This deduplicates objects similar to `String.intern()`, but for arbitrary objects. */ public interface Interner { + /** + * Intern the given object, returning an existing object if one exists. Returns `null` if no such object exists. + * This is intended for use in cases where the caller needs to know whether the object was interned or not. + */ + T maybeIntern(T sample); + /** * Intern the given object, returning an existing object if one exists. */ - T intern(T sample); + default T intern(T sample) { + T interned = maybeIntern(sample); + if (interned != null) { + return interned; + } + return sample; + } } diff --git a/chunky/src/java/se/llbit/util/interner/StrongInterner.java b/chunky/src/java/se/llbit/util/interner/StrongInterner.java index 3651fec74f..7899cbd273 100644 --- a/chunky/src/java/se/llbit/util/interner/StrongInterner.java +++ b/chunky/src/java/se/llbit/util/interner/StrongInterner.java @@ -26,6 +26,16 @@ public class StrongInterner implements Interner { protected final HashMap pool = new HashMap<>(); + @Override + public T maybeIntern(T sample) { + T interned = pool.get(sample); + if (interned != null) { + return interned; + } + pool.put(sample, sample); + return null; + } + @Override public T intern(T sample) { return pool.computeIfAbsent(sample, k -> k); diff --git a/chunky/src/java/se/llbit/util/interner/WeakInterner.java b/chunky/src/java/se/llbit/util/interner/WeakInterner.java index 9ed0b406eb..6584acc036 100644 --- a/chunky/src/java/se/llbit/util/interner/WeakInterner.java +++ b/chunky/src/java/se/llbit/util/interner/WeakInterner.java @@ -59,7 +59,7 @@ public boolean equals(Object obj) { protected final ReferenceQueue queue = new ReferenceQueue<>(); @Override - public T intern(T sample) { + public T maybeIntern(T sample) { compact(); HashedWeakReference ref = new HashedWeakReference<>(sample, queue); @@ -75,7 +75,7 @@ public T intern(T sample) { // Existing object is dead or doesn't exist // Enqueue our new ref pool.put(ref, ref); - return sample; + return null; } } diff --git a/chunky/src/test/se/llbit/util/WeakInternerTest.java b/chunky/src/test/se/llbit/util/WeakInternerTest.java index df8237188c..c26721988f 100644 --- a/chunky/src/test/se/llbit/util/WeakInternerTest.java +++ b/chunky/src/test/se/llbit/util/WeakInternerTest.java @@ -19,9 +19,9 @@ package se.llbit.util; import org.junit.Assume; -import org.junit.Ignore; import org.junit.Test; import se.llbit.util.interner.Interner; +import se.llbit.util.interner.StrongInterner; import se.llbit.util.interner.WeakInterner; import java.lang.ref.WeakReference; @@ -29,7 +29,6 @@ import java.util.Random; import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; public class WeakInternerTest { private static class TestInternable { @@ -54,9 +53,17 @@ public int hashCode() { } @Test - public void testSmoke() { + public void testWeakInternerSmokeIntern() { + smokeIntern(new WeakInterner<>()); + } + + @Test + public void testStrongInternerSmokeIntern() { + smokeIntern(new StrongInterner<>()); + } + + private void smokeIntern(Interner interner) { Random rand = new Random(0); - Interner interner = new WeakInterner<>(); TestInternable test11 = new TestInternable(rand.nextLong()); TestInternable test12 = new TestInternable(rand.nextLong()); @@ -76,6 +83,37 @@ public void testSmoke() { assertSame(interner.intern(test23), test13); } + @Test + public void testWeakInternerSmokeMaybeIntern() { + smokeMaybeIntern(new WeakInterner<>()); + } + + @Test + public void testStrongInternerSmokeMaybeIntern() { + smokeMaybeIntern(new StrongInterner<>()); + } + + private void smokeMaybeIntern(Interner interner) { + Random rand = new Random(0); + + TestInternable test11 = new TestInternable(rand.nextLong()); + TestInternable test12 = new TestInternable(rand.nextLong()); + TestInternable test13 = new TestInternable(rand.nextLong()); + TestInternable test21 = new TestInternable(test11.value); + TestInternable test22 = new TestInternable(test12.value); + TestInternable test23 = new TestInternable(test13.value); + + // Test that we get `null` when we maybeIntern the object. + assertSame(interner.maybeIntern(test11), null); + assertSame(interner.maybeIntern(test12), null); + assertSame(interner.maybeIntern(test13), null); + + // Test that we get back the interned object when interning the copy + assertSame(interner.maybeIntern(test21), test11); + assertSame(interner.maybeIntern(test22), test12); + assertSame(interner.maybeIntern(test23), test13); + } + @Test public void testWeakness() { Random rand = new Random(0); From 1a6bc55ad3ec09ef3adb86155966e0f6cf329b74 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 18 Sep 2023 21:23:25 -0700 Subject: [PATCH 5/7] World texture 3d. --- .../renderer/scene/biome/BiomeStructure.java | 2 + .../biome/worldtexture/ChunkTexture.java | 9 ++ .../biome/worldtexture/ChunkTexture3d.java | 145 +++++++++++++++++ .../WorldTexture2dBiomeStructure.java | 18 +-- .../WorldTexture3dBiomeStructure.java | 153 ++++++++++++++++++ 5 files changed, 311 insertions(+), 16 deletions(-) create mode 100644 chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture3d.java create mode 100644 chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java index 205e4f1e43..cef16a7ede 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java @@ -2,6 +2,7 @@ import it.unimi.dsi.fastutil.objects.Object2ReferenceOpenHashMap; import se.llbit.chunky.renderer.scene.biome.worldtexture.WorldTexture2dBiomeStructure; +import se.llbit.chunky.renderer.scene.biome.worldtexture.WorldTexture3dBiomeStructure; import se.llbit.chunky.world.Chunk; import se.llbit.log.Log; import se.llbit.math.structures.Position2IntStructure; @@ -25,6 +26,7 @@ static void registerDefaults() { BiomeStructure.register(new Trivial3dBiomeStructure()); BiomeStructure.register(new Trivial2dBiomeStructure()); BiomeStructure.register(new WorldTexture2dBiomeStructure()); + BiomeStructure.register(new WorldTexture3dBiomeStructure.Factory()); } /** diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java index 9345267294..993b08b59c 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java @@ -20,6 +20,7 @@ import se.llbit.chunky.world.Chunk; import se.llbit.math.ColorUtil; +import se.llbit.util.interner.Interner; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -74,6 +75,14 @@ public void makeReadOnly() { writeable = false; } + /** + * Intern this chunk texture. + */ + public ChunkTexture intern(Interner interner) { + this.makeReadOnly(); + return interner.intern(this); + } + /** * Set the color value at (x, z). * @param frgb RGB color components to set diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture3d.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture3d.java new file mode 100644 index 0000000000..11ae8f2cc2 --- /dev/null +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture3d.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2023 Chunky contributors + * + * This file is part of Chunky. + * + * Chunky is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chunky is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with Chunky. If not, see . + */ + +package se.llbit.chunky.renderer.scene.biome.worldtexture; + +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import se.llbit.util.interner.Interner; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +public class ChunkTexture3d { + protected final Int2ObjectOpenHashMap data = new Int2ObjectOpenHashMap<>(); + protected transient boolean writeable = true; + + /** + * Create new texture + */ + public ChunkTexture3d() { + } + + /** + * Copy an existing chunk texture + */ + public ChunkTexture3d(ChunkTexture3d ct) { + for (Int2ObjectMap.Entry entry : ct.data.int2ObjectEntrySet()) { + data.put(entry.getIntKey(), new ChunkTexture(entry.getValue())); + } + } + + /** + * Load a chunk texture from an input stream. + */ + public static ChunkTexture3d load(DataInputStream in, Interner interner) throws IOException { + ChunkTexture3d tex = new ChunkTexture3d(); + int count = in.readInt(); + for (int i = 0; i < count; i++) { + int y = in.readInt(); + ChunkTexture ct = ChunkTexture.load(in).intern(interner); + tex.data.put(y, ct); + } + return tex; + } + + /** + * Write this chunk texture to an output stream. + */ + public void store(DataOutputStream out) throws IOException { + out.writeInt(data.size()); + for (Int2ObjectMap.Entry entry : data.int2ObjectEntrySet()) { + out.writeInt(entry.getIntKey()); + entry.getValue().store(out); + } + } + + /** + * Make this chunk texture read-only. + */ + public void makeReadOnly() { + writeable = false; + } + + /** + * Intern this chunk texture. + */ + public ChunkTexture3d intern(Interner interner) { + this.makeReadOnly(); + return interner.intern(this); + } + + /** + * Compact this chunk texture. + */ + public void compact(Interner interner) { + for (Int2ObjectMap.Entry entry : data.int2ObjectEntrySet()) { + entry.setValue(entry.getValue().intern(interner)); + } + } + + /** + * Set the color value at (x, y, z). + */ + public ChunkTexture3d set(int x, int y, int z, float[] frgb) { + if (writeable) { + ChunkTexture ct = data.get(y); + if (ct == null) { + ct = new ChunkTexture(); + } + + ct = ct.set(x, z, frgb); + data.put(y, ct); + return this; + } else { + ChunkTexture3d tex = new ChunkTexture3d(this); + return tex.set(x, y, z, frgb); + } + } + + /** + * Get the color value at (x, y, z). + */ + public float[] get(int x, int y, int z) { + ChunkTexture ct = data.get(y); + if (ct == null) { + return null; + } + return ct.get(x, z); + } + + @Override + public int hashCode() { + int hashCode = 1; + for (Int2ObjectMap.Entry entry : data.int2ObjectEntrySet()) { + hashCode = 31 * hashCode + entry.getIntKey(); + hashCode = 31 * hashCode + entry.getValue().hashCode(); + } + return hashCode; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ChunkTexture3d)) { + return false; + } + ChunkTexture3d other = (ChunkTexture3d) obj; + return data.equals(other.data); + } +} diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java index da3ccff576..411324beb9 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java @@ -76,15 +76,7 @@ public static Impl load(DataInputStream in) throws IOException { for (int i = 0; i < numTiles; i++) { int x = in.readInt(); int z = in.readInt(); - ChunkTexture tile = ChunkTexture.load(in); - - ChunkTexture interned = interner.maybeIntern(tile); - if (interned != null) { - interned.makeReadOnly(); - tile = interned; - } - - tile = interner.intern(tile); + ChunkTexture tile = ChunkTexture.load(in).intern(interner); texture.map.put(chunkPos(x, z), tile); } @@ -108,13 +100,7 @@ public void store(DataOutputStream out) throws IOException { public void endFinalization() { Interner interner = new StrongInterner<>(); for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { - ChunkTexture interned = interner.maybeIntern(entry.getValue()); - - // We did de-duplicate, mark the texture as shared and replace the value - if (interned != null) { - interned.makeReadOnly(); - entry.setValue(interned); - } + entry.setValue(entry.getValue().intern(interner)); } } diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java new file mode 100644 index 0000000000..ceef621150 --- /dev/null +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2023 Chunky contributors + * + * This file is part of Chunky. + * + * Chunky is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chunky is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with Chunky. If not, see . + */ + +package se.llbit.chunky.renderer.scene.biome.worldtexture; + +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import se.llbit.chunky.renderer.scene.biome.BiomeStructure; +import se.llbit.util.annotation.NotNull; +import se.llbit.util.interner.Interner; +import se.llbit.util.interner.StrongInterner; +import se.llbit.util.interner.WeakInterner; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +public class WorldTexture3dBiomeStructure implements BiomeStructure { + public static final String ID = "WORLD_TEXTURE_3D"; + + private final Long2ObjectOpenHashMap map = new Long2ObjectOpenHashMap<>(); + + private static long chunkPos(int x, int z) { + return (((long) x) << 32) | ((long) z); + } + + public static WorldTexture3dBiomeStructure load(DataInputStream in) throws IOException { + WorldTexture3dBiomeStructure texture = new WorldTexture3dBiomeStructure(); + Interner ct3dInterner = new StrongInterner<>(); + Interner ct2dInterner = new StrongInterner<>(); + + int numTiles = in.readInt(); + for (int i = 0; i < numTiles; i++) { + int x = in.readInt(); + int z = in.readInt(); + + ChunkTexture3d column = ChunkTexture3d.load(in, ct2dInterner); + texture.map.put(chunkPos(x, z), column); + } + + return texture; + } + + @Override + public void store(DataOutputStream out) throws IOException { + out.writeInt(map.size()); + for (Long2ObjectOpenHashMap.Entry entry : map.long2ObjectEntrySet()) { + long pos = entry.getLongKey(); + ChunkTexture3d texture = entry.getValue(); + + int x = (int) (pos >> 32); + int z = (int) pos; + out.writeInt(x); + out.writeInt(z); + texture.store(out); + } + } + + @Override + public void endFinalization() { + Interner interner3D = new StrongInterner<>(); + Interner interner2D = new StrongInterner<>(); + for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { + ChunkTexture3d value = entry.getValue(); + value.makeReadOnly(); + + ChunkTexture3d interned = interner3D.maybeIntern(value); + if (interned != null) { + entry.setValue(interned); + } else { + value.compact(interner2D); + } + } + } + + @Override + public String biomeFormat() { + return ID; + } + + @Override + public void set(int x, int y, int z, float[] data) { + long cp = chunkPos(x >> 4, z >> 4); + ChunkTexture3d texture = map.get(cp); + + if (texture == null) { + texture = new ChunkTexture3d(); + map.put(cp, texture); + } + + ChunkTexture3d newTex = texture.set(x & 0xF, y, z & 0xF, data); + if (newTex != texture) { + map.put(cp, newTex); + } + } + + @Override + public float[] get(int x, int y, int z) { + ChunkTexture3d texture = map.get(chunkPos(x >> 4, z >> 4)); + if (texture == null) { + return null; + } + return texture.get(x & 0xF, y, z & 0xF); + } + + public static class Factory implements BiomeStructure.Factory { + + @Override + public BiomeStructure create() { + return new WorldTexture3dBiomeStructure(); + } + + @Override + public BiomeStructure load(@NotNull DataInputStream in) throws IOException { + return WorldTexture3dBiomeStructure.load(in); + } + + @Override + public boolean is3d() { + return true; + } + + @Override + public String getName() { + return "World Texture 3D"; + } + + @Override + public String getDescription() { + return "A 3d biome format that uses de-duplicated bitmaps per chunk."; + } + + @Override + public String getId() { + return ID; + } + } +} From 35e5ed193462b3e2c20f88e6269f94f63d6ae36a Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 19 Sep 2023 21:30:33 -0700 Subject: [PATCH 6/7] Refactor intern functions to be better. Implement compacting after loading each chunk. --- .../biome/worldtexture/ChunkTexture.java | 3 + .../biome/worldtexture/ChunkTexture3d.java | 19 ++++-- .../WorldTexture3dBiomeStructure.java | 64 ++++++++++++++----- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java index 993b08b59c..6be902f799 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture.java @@ -27,6 +27,9 @@ import java.io.IOException; import java.util.Arrays; +/** + * A 2D chunk texture. This stores a pixel for each block in a chunk, with 8 bits per channel. + */ public class ChunkTexture { protected final byte[] data = new byte[Chunk.X_MAX * Chunk.Z_MAX * 3]; protected transient boolean writeable = true; diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture3d.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture3d.java index 11ae8f2cc2..1532e3a1d2 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture3d.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/ChunkTexture3d.java @@ -26,6 +26,10 @@ import java.io.DataOutputStream; import java.io.IOException; +/** + * A 3D chunk texture. This stores a column of {@link ChunkTexture}'s, each corresponding to a different + * y-level in a chunk. + */ public class ChunkTexture3d { protected final Int2ObjectOpenHashMap data = new Int2ObjectOpenHashMap<>(); protected transient boolean writeable = true; @@ -48,12 +52,12 @@ public ChunkTexture3d(ChunkTexture3d ct) { /** * Load a chunk texture from an input stream. */ - public static ChunkTexture3d load(DataInputStream in, Interner interner) throws IOException { + public static ChunkTexture3d load(DataInputStream in) throws IOException { ChunkTexture3d tex = new ChunkTexture3d(); int count = in.readInt(); for (int i = 0; i < count; i++) { int y = in.readInt(); - ChunkTexture ct = ChunkTexture.load(in).intern(interner); + ChunkTexture ct = ChunkTexture.load(in); tex.data.put(y, ct); } return tex; @@ -80,9 +84,16 @@ public void makeReadOnly() { /** * Intern this chunk texture. */ - public ChunkTexture3d intern(Interner interner) { + public ChunkTexture3d intern(Interner interner2d, Interner interner3d) { this.makeReadOnly(); - return interner.intern(this); + + ChunkTexture3d interned = interner3d.intern(this); + if (interned == this) { + // This dosen't actually modify the visible data of this CT so it's safe to do despite being interned already + this.compact(interner2d); + } + + return interned; } /** diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java index ceef621150..249f905350 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java @@ -18,8 +18,11 @@ package se.llbit.chunky.renderer.scene.biome.worldtexture; +import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import se.llbit.chunky.renderer.scene.biome.BiomeStructure; import se.llbit.util.annotation.NotNull; import se.llbit.util.interner.Interner; @@ -33,23 +36,43 @@ public class WorldTexture3dBiomeStructure implements BiomeStructure { public static final String ID = "WORLD_TEXTURE_3D"; + /** + * Map from chunk position to chunk texture. + */ private final Long2ObjectOpenHashMap map = new Long2ObjectOpenHashMap<>(); + /** + * List of live, un-interned chunk textures. + */ + private LongOpenHashSet live = null; + private Interner interner2d = null; + private Interner interner3d = null; + + protected WorldTexture3dBiomeStructure(boolean doIntern) { + if (doIntern) { + live = new LongOpenHashSet(); + interner2d = new StrongInterner<>(); + interner3d = new StrongInterner<>(); + } + } + private static long chunkPos(int x, int z) { return (((long) x) << 32) | ((long) z); } public static WorldTexture3dBiomeStructure load(DataInputStream in) throws IOException { - WorldTexture3dBiomeStructure texture = new WorldTexture3dBiomeStructure(); - Interner ct3dInterner = new StrongInterner<>(); - Interner ct2dInterner = new StrongInterner<>(); + // Set doIntern to false since we can do it better + WorldTexture3dBiomeStructure texture = new WorldTexture3dBiomeStructure(false); + + Interner interner2d = new StrongInterner<>(); + Interner interner3d = new StrongInterner<>(); int numTiles = in.readInt(); for (int i = 0; i < numTiles; i++) { int x = in.readInt(); int z = in.readInt(); - ChunkTexture3d column = ChunkTexture3d.load(in, ct2dInterner); + ChunkTexture3d column = ChunkTexture3d.load(in).intern(interner2d, interner3d); texture.map.put(chunkPos(x, z), column); } @@ -71,21 +94,28 @@ public void store(DataOutputStream out) throws IOException { } } + @Override + public void compact() { + if (live == null || interner2d == null || interner3d == null) { + return; + } + + for (long cp : live) { + map.computeIfPresent(cp, (k, v) -> v.intern(interner2d, interner3d)); + } + live.clear(); + } + @Override public void endFinalization() { - Interner interner3D = new StrongInterner<>(); - Interner interner2D = new StrongInterner<>(); for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { ChunkTexture3d value = entry.getValue(); - value.makeReadOnly(); - - ChunkTexture3d interned = interner3D.maybeIntern(value); - if (interned != null) { - entry.setValue(interned); - } else { - value.compact(interner2D); - } + entry.setValue(value.intern(interner2d, interner3d)); } + + live = null; + interner2d = null; + interner3d = null; } @Override @@ -107,6 +137,10 @@ public void set(int x, int y, int z, float[] data) { if (newTex != texture) { map.put(cp, newTex); } + + if (live != null) { + live.add(cp); + } } @Override @@ -122,7 +156,7 @@ public static class Factory implements BiomeStructure.Factory { @Override public BiomeStructure create() { - return new WorldTexture3dBiomeStructure(); + return new WorldTexture3dBiomeStructure(true); } @Override From 0146388fd646ab8f07d3a138a82e36d756106b14 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 22 Sep 2023 10:22:57 -0700 Subject: [PATCH 7/7] Refactor WorldTexture2dBiomeStructure --- .../renderer/scene/biome/BiomeStructure.java | 2 +- .../WorldTexture2dBiomeStructure.java | 135 +++++++++--------- .../WorldTexture3dBiomeStructure.java | 6 - 3 files changed, 70 insertions(+), 73 deletions(-) diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java index cef16a7ede..1b3f57bdde 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/BiomeStructure.java @@ -25,7 +25,7 @@ static void registerDefaults() { //TODO: create a plugin api interface for registering implementations, and move this to that BiomeStructure.register(new Trivial3dBiomeStructure()); BiomeStructure.register(new Trivial2dBiomeStructure()); - BiomeStructure.register(new WorldTexture2dBiomeStructure()); + BiomeStructure.register(new WorldTexture2dBiomeStructure.Factory()); BiomeStructure.register(new WorldTexture3dBiomeStructure.Factory()); } diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java index 411324beb9..e17c694d02 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture2dBiomeStructure.java @@ -21,6 +21,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import se.llbit.chunky.renderer.scene.biome.BiomeStructure; +import se.llbit.util.annotation.NotNull; import se.llbit.util.interner.Interner; import se.llbit.util.interner.StrongInterner; @@ -28,109 +29,111 @@ import java.io.DataOutputStream; import java.io.IOException; -public class WorldTexture2dBiomeStructure implements BiomeStructure.Factory { +public class WorldTexture2dBiomeStructure implements BiomeStructure { public static final String ID = "WORLD_TEXTURE_2D"; - @Override - public BiomeStructure create() { - return new Impl(); - } + private final Long2ObjectOpenHashMap map = new Long2ObjectOpenHashMap<>(); - @Override - public BiomeStructure load(DataInputStream in) throws IOException { - return Impl.load(in); + private static long chunkPos(int x, int z) { + return (((long) x) << 32) | ((long) z); } - @Override - public boolean is3d() { - return false; + public static WorldTexture2dBiomeStructure load(DataInputStream in) throws IOException { + WorldTexture2dBiomeStructure texture = new WorldTexture2dBiomeStructure(); + Interner interner = new StrongInterner<>(); + + int numTiles = in.readInt(); + for (int i = 0; i < numTiles; i++) { + int x = in.readInt(); + int z = in.readInt(); + ChunkTexture tile = ChunkTexture.load(in).intern(interner); + texture.map.put(chunkPos(x, z), tile); + } + + return texture; } @Override - public String getName() { - return "World texture 2d"; + public void store(DataOutputStream out) throws IOException { + out.writeInt(map.size()); + for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { + long pos = entry.getLongKey(); + ChunkTexture texture = entry.getValue(); + + out.writeInt((int) (pos >> 32)); + out.writeInt((int) pos); + texture.store(out); + } } @Override - public String getDescription() { - return "A 2d biome format that uses de-duplicated bitmaps per chunk."; + public void endFinalization() { + Interner interner = new StrongInterner<>(); + for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { + entry.setValue(entry.getValue().intern(interner)); + } } @Override - public String getId() { + public String biomeFormat() { return ID; } - static class Impl implements BiomeStructure { - private final Long2ObjectOpenHashMap map = new Long2ObjectOpenHashMap<>(); + @Override + public void set(int x, int y, int z, float[] data) { + long cp = chunkPos(x >> 4, z >> 4); + ChunkTexture texture = map.get(cp); - private static long chunkPos(int x, int z) { - return (((long) x) << 32) | ((long) z); + if (texture == null) { + texture = new ChunkTexture(); + map.put(cp, texture); } - public static Impl load(DataInputStream in) throws IOException { - Impl texture = new Impl(); - Interner interner = new StrongInterner<>(); + ChunkTexture newTex = texture.set(x & 0xF, z & 0xF, data); + if (newTex != texture) { + map.put(cp, newTex); + } + } - int numTiles = in.readInt(); - for (int i = 0; i < numTiles; i++) { - int x = in.readInt(); - int z = in.readInt(); - ChunkTexture tile = ChunkTexture.load(in).intern(interner); - texture.map.put(chunkPos(x, z), tile); - } + @Override + public float[] get(int x, int y, int z) { + ChunkTexture texture = map.get(chunkPos(x >> 4, z >> 4)); + if (texture == null) { + return null; + } + return texture.get(x & 0xF, z & 0xF); + } - return texture; + + public static class Factory implements BiomeStructure.Factory { + @Override + public BiomeStructure create() { + return new WorldTexture2dBiomeStructure(); } @Override - public void store(DataOutputStream out) throws IOException { - out.writeInt(map.size()); - for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { - long pos = entry.getLongKey(); - ChunkTexture texture = entry.getValue(); - - out.writeInt((int) (pos >> 32)); - out.writeInt((int) pos); - texture.store(out); - } + public BiomeStructure load(@NotNull DataInputStream in) throws IOException { + return WorldTexture2dBiomeStructure.load(in); } @Override - public void endFinalization() { - Interner interner = new StrongInterner<>(); - for (Long2ObjectMap.Entry entry : map.long2ObjectEntrySet()) { - entry.setValue(entry.getValue().intern(interner)); - } + public boolean is3d() { + return false; } @Override - public String biomeFormat() { - return ID; + public String getName() { + return "World texture 2d"; } @Override - public void set(int x, int y, int z, float[] data) { - long cp = chunkPos(x >> 4, z >> 4); - ChunkTexture texture = map.get(cp); - - if (texture == null) { - texture = new ChunkTexture(); - } - - ChunkTexture newTex = texture.set(x & 0xF, z & 0xF, data); - if (newTex != texture) { - map.put(cp, newTex); - } + public String getDescription() { + return "A 2d biome format that uses de-duplicated bitmaps per chunk."; } @Override - public float[] get(int x, int y, int z) { - ChunkTexture texture = map.get(chunkPos(x >> 4, z >> 4)); - if (texture == null) { - return null; - } - return texture.get(x & 0xF, z & 0xF); + public String getId() { + return ID; } } } diff --git a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java index 249f905350..d7a9061c96 100644 --- a/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java +++ b/chunky/src/java/se/llbit/chunky/renderer/scene/biome/worldtexture/WorldTexture3dBiomeStructure.java @@ -18,16 +18,13 @@ package se.llbit.chunky.renderer.scene.biome.worldtexture; -import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; -import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import se.llbit.chunky.renderer.scene.biome.BiomeStructure; import se.llbit.util.annotation.NotNull; import se.llbit.util.interner.Interner; import se.llbit.util.interner.StrongInterner; -import se.llbit.util.interner.WeakInterner; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -41,9 +38,6 @@ public class WorldTexture3dBiomeStructure implements BiomeStructure { */ private final Long2ObjectOpenHashMap map = new Long2ObjectOpenHashMap<>(); - /** - * List of live, un-interned chunk textures. - */ private LongOpenHashSet live = null; private Interner interner2d = null; private Interner interner3d = null;