Merge pull request #112 from ferrumc-rs/rewrite/db

Database importing

.etc/example-config.toml

@@ -0,0 +1,31 @@
+# IP or hostname where the server will bind. Leave as 0.0.0.0 if you want it to be accessible from any IP.
+host = "0.0.0.0"
+# Server port (0-65535). Minecraft's default port is 25565, so you probably want to leave it as is.
+port = 25565
+# Message of the day. A random one will be selected.
+motd = ["Welcome to the best server ever!", "Rust", "Good luck, have fun!"]
+# Maximum number of players
+max_players = 100
+# How many network updates per second per user
+network_tick_rate = 30
+# World name to load
+world = "world"
+# Network compression threshold (can be negative). This decides how long a packet has to be before it is compressed.
+# Very small packets may actually increase in size when compressed, so setting it to 0 won't be perfect in all situations.
+# Set to -1 to disable compression.
+network_compression_threshold = 64
+
+# Database configuration
+[database]
+# Cache size in KB
+cache_size = 1024
+# Compression algorithm (brotli, deflate, gzip, zlib, zstd)
+compression = "gzip"
+# Database backend (redb, rocksdb, sled, surrealkv)
+backend = "redb"
+# Path to the world database
+db_path = "/path/to/world/database"
+# Path to world import folder (e.g., %APPDATA%/.minecraft/saves/world)
+import_path = "/path/to/import/folder"
+# Compression level (0-22) for the database. Higher values mean less disk space but will take longer to read/write.
+compression_level = 5

.gitignore

@@ -6,3 +6,9 @@
 Cargo.lock
 config.toml
 .temp
+
+.etc/blockstates.json
+
+.etc/blocks.json
+
+flame.svg

Cargo.toml

@@ -86,7 +86,7 @@ ferrumc-general-purpose = { path = "src/lib/utils/general_purpose" }
 ferrumc-macros = { path = "src/lib/derive_macros" }
 ferrumc-world = { path = "src/lib/world" }
 ferrumc-nbt = { path = "src/lib/adapters/nbt" }
-ferrumc-mca = { path = "src/lib/adapters/mca" }
+ferrumc-anvil = { path = "src/lib/adapters/anvil" }
 ferrumc-tests = { path = "src/tests" }
 
 
@@ -114,12 +114,15 @@ thiserror = "1.0.63"
 # Cryptography
 rsa = "0.9.6"
 rand = "0.9.0-alpha.2"
+fnv = "1.0.7"
 
 # Encoding/Serialization
 serde = "1.0.210"
 serde_json = "1.0.128"
 serde_derive = "1.0.210"
 base64 = "0.22.1"
+bitcode = "0.6.3"
+bitcode_derive = "0.6.3"
 
 # Data types
 hashbrown = "0.15.0"
@@ -134,6 +137,7 @@ syn = "2.0.77"
 proc-macro2 = "1.0.86"
 proc-macro-crate = "3.2.0"
 maplit = "1.0.2"
+macro_rules_attribute = "0.2.0"
 
 # Magic ("life-before-main" initialization, __attribute__((constructor)))
 ctor = "0.2.8"
@@ -145,10 +149,28 @@ zstd = { version = "0.13.2" }
 brotli = "7.0.0"
 lzzzz = "1.1.0"
 yazi = "0.2.0"
+bzip2 = "0.4.1"
+
+# CLI
+clap = "4.5.20"
+indicatif = "0.17.8"
+
+# Misc
+deepsize = "0.2.0"
 
 # I/O
 tempfile = "3.12.0"
 memmap2 = "0.9.5"
 
 # Benchmarking
 criterion = { version = "0.5.1", features = ["html_reports"] }
+
+[profile.hyper]
+inherits = "release"
+lto = true
+opt-level = 3
+debug = false
+debug-assertions = false
+overflow-checks = false
+panic = "abort"
+codegen-units = 1

scripts/block_parser.py

@@ -0,0 +1,32 @@
+import bz2
+import json
+
+
+def dict_reorder(item):
+    return {k: dict_reorder(v) if isinstance(v, dict) else v for k, v in sorted(item.items())}
+
+out = {}
+
+with open("../.etc/blocks.json") as f:
+    blocks = json.load(f)
+    for block in blocks:
+        data = blocks[block]
+        for state in data["states"]:
+            if "properties" in state:
+                props = state["properties"]
+                if "id" in state:
+                    block_id = state["id"]
+                    out[block_id] = {"name": block, "properties": props}
+            else:
+                block_id = state["id"]
+                out[block_id] = {"name": block, "default": True}
+
+out = dict_reorder(out)
+
+with open("../.etc/blockstates.json", "w") as bs:
+    json.dump(out, bs, indent=4)
+with open("../.etc/blockmappings.bz2", "wb") as f:
+    as_string = json.dumps(out, separators=(',', ':'))
+    print("Compressing and writing to file...")
+    print("This may take a while.")
+    f.write(bz2.compress(as_string.encode("utf-8")))

scripts/compression_bencher.py

@@ -0,0 +1,59 @@
+import bz2
+import gzip
+import lzma
+import time
+import zlib
+
+results = {}
+
+def compress_zlib_max(data):
+    return zlib.compress(data, 9)
+
+def compress_zlib_min(data):
+    return zlib.compress(data, 1)
+
+def compress_gzip_max(data):
+    return gzip.compress(data, 5)
+
+def compress_gzip_min(data):
+    return gzip.compress(data, 1)
+
+def compress_lzma(data):
+    return lzma.compress(data)
+
+def compress_bzip2_max(data):
+    return bz2.compress(data, 9)
+
+def compress_bzip2_min(data):
+    return bz2.compress(data, 1)
+
+def compress_none(data):
+    return data
+
+
+with open("../.etc/codec.nbt", "rb") as f:
+    data = f.read()
+    original_size = len(data)
+    for algo in [compress_zlib_max, compress_zlib_min, compress_gzip_max, compress_gzip_min, compress_lzma, compress_bzip2_max, compress_bzip2_min, compress_none]:
+        print(f"Testing {algo.__name__}...")
+        times = []
+        sizediff = 0
+        for iteration in range(1, 3000):
+            start = time.time_ns()
+            compressed = algo(data)
+            end = time.time_ns()
+            times.append(end - start)
+            sizediff = len(compressed) / original_size
+        average_time = (sum(times) / len(times) / 1000) / 1000
+        results[algo.__name__] = {
+            "average_time": average_time,
+            "size_decrease_percentage": sizediff
+        }
+    for result in results:
+        name = result
+        average_time = results[result]["average_time"]
+        sizediff = results[result]["size_decrease_percentage"]
+        print(f"Algorithm: {name}")
+        print(f"Average Time: {average_time:.3f} ms")
+        print(f"Compressed Size: {sizediff*100:.2f}% of original size")
+        print()

scripts/json_minimiser.py

@@ -0,0 +1,8 @@
+import json
+
+path = input("Enter the path to the file: ")
+jsondata = ""
+with open(path, "r") as f:
+    jsondata = json.load(f)
+with open(path, "w") as f:
+    json.dump(jsondata, f, separators=(',', ':'))

src/bin/Cargo.toml

@@ -39,7 +39,8 @@ tokio = { workspace = true }
 rayon = { workspace = true }
 futures = { workspace = true }
 serde_json = { workspace = true }
-async-trait = "0.1.83"
+async-trait = { workspace = true }
+clap = { workspace = true, features = ["derive"] }
 
 
 [[bin]]

src/bin/src/main.rs

@@ -2,12 +2,23 @@
 #![forbid(unsafe_code)]
 extern crate core;
 
+use std::path::PathBuf;
 use ferrumc_ecs::Universe;
 use ferrumc_net::server::create_server_listener;
 use ferrumc_net::ServerState;
 use std::sync::Arc;
+use clap::Parser;
 use systems::definition;
 use tracing::{error, info};
+use ferrumc_config::statics::get_global_config;
+use ferrumc_world::World;
+
+#[derive(clap::Parser)]
+struct CLIArgs {
+    #[clap(long)]
+    import: bool
+}
+
 
 pub(crate) mod errors;
 mod packet_handlers;
@@ -20,6 +31,21 @@ async fn main() {
     ferrumc_logging::init_logging();
 
     println!("good day to ya. enjoy your time with ferrumc!");
+
+    let cli_args = CLIArgs::parse();
+
+    if cli_args.import {
+        // Import the world
+        let config = get_global_config();
+        let mut world = World::new().await;
+        let import_path = PathBuf::from(config.database.import_path.clone());
+        let db_path = PathBuf::from(config.database.db_path.clone());
+        if let Err(e) = world.import(import_path, db_path).await {
+            error!("Could not import world: {:?}", e);
+            return;
+        }
+        return;
+    }
 
     if let Err(e) = entry().await {
         error!("Server exited with the following error;");

src/lib/adapters/anvil/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "anvil"
+name = "ferrumc-anvil"
 version = "0.1.0"
 edition = "2021"
 

src/lib/adapters/anvil/benches/anvil.rs

@@ -1,10 +1,10 @@
 use std::fs::File;
 use std::path::PathBuf;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use anvil::load_anvil_file;
 use ferrumc_utils::root;
 use rayon::prelude::*;
 use fastanvil::Region;
+use ferrumc_anvil::load_anvil_file;
 
 fn criterion_benchmark(c: &mut Criterion) {
     let mut read_all_group = c.benchmark_group("Read All");

src/lib/adapters/anvil/src/lib.rs

@@ -1,4 +1,4 @@
-mod errors;
+pub mod errors;
 
 use std::io::Read;
 use std::path::PathBuf;
@@ -37,7 +37,8 @@ pub fn get_chunk(x: u32, z: u32, file_path: PathBuf) -> Option<Vec<u8>> {
 /// ```no_run
 /// use std::fs::File;
 /// use fastanvil::Region;
-/// use std::path::PathBuf;use anvil::load_anvil_file;
+/// use std::path::PathBuf;
+/// use ferrumc_anvil::load_anvil_file;
 ///
 /// let file_path = PathBuf::from("r.0.0.mca");
 ///

src/lib/net/crates/codec/Cargo.toml

@@ -6,4 +6,5 @@ edition = "2021"
 [dependencies]
 thiserror = { workspace = true }
 tokio = { workspace = true }
-ferrumc-config = { workspace = true }
+ferrumc-config = { workspace = true }
+bitcode = { workspace = true }

src/lib/net/crates/codec/src/net_types/var_int.rs

@@ -3,10 +3,11 @@ use crate::decode::{NetDecode, NetDecodeOpts, NetDecodeResult};
 use crate::encode::errors::NetEncodeError;
 use crate::encode::{NetEncode, NetEncodeOpts, NetEncodeResult};
 use std::io::{Read, Write};
+use bitcode::{Decode, Encode};
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use crate::net_types::NetTypesError;
 
-#[derive(Debug)]
+#[derive(Debug, Encode, Decode)]
 pub struct VarInt {
     /// The value of the VarInt.
     pub val: i32,

src/lib/storage/Cargo.toml

@@ -18,9 +18,11 @@ tokio = { workspace = true }
 parking_lot = { workspace = true }
 lazy_static = { workspace = true }
 redb = { version = "2.1.3", optional = true }
-surrealkv = { version = "0.3.6", optional = true }
+surrealkv = { version = "0.4.0", optional = true }
 sled = { version = "0.34.7" , optional = true }
 rocksdb = { version = "0.22.0" , optional = true }
+async-trait = { workspace = true}
+yazi = { workspace = true}
 
 [features]
 rocksdb = ["dep:rocksdb"]

src/lib/storage/src/backends/mod.rs

@@ -9,5 +9,20 @@ pub mod sled;
 #[cfg(feature = "surrealkv")]
 pub mod surrealkv;
 
-#[cfg(not(any(feature = "redb", feature = "rocksdb", feature = "sled", feature = "surrealkv")))]
+#[cfg(not(any(
+    feature = "redb",
+    feature = "rocksdb",
+    feature = "sled",
+    feature = "surrealkv"
+)))]
 compile_error!("At least one storage backend must be enabled");
+
+pub fn backend_enabled(backend: &str) -> bool {
+    match backend {
+        "redb" => cfg!(feature = "redb"),
+        "rocksdb" => cfg!(feature = "rocksdb"),
+        "sled" => cfg!(feature = "sled"),
+        "surrealkv" => cfg!(feature = "surrealkv"),
+        _ => false,
+    }
+}