From c7286d8f3d8b2ab827f3aebafa18416052accea9 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Tue, 15 Mar 2011 15:31:53 -0500 Subject: [PATCH 01/40] Syntax error --- nbt/nbt.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/nbt/nbt.py b/nbt/nbt.py index d1229bd..f534845 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -396,13 +396,12 @@ def write_chunk(self, x, z, nbt_file): self.file.seek(0) found = True for intersect_offset, intersect_len in ( (extent_offset, extent_len) - for extent_offset, extent_len - in (unpack(">IB", "\0"+self.file.read(4)) for block in xrange(1024)) - if extent_offset != 0 and ( sector >= extent_offset < (sector+nsectors))): - #move foward to end of intersect - sector = intersect_offset + intersect_len - found = False - break + for extent_offset, extent_len in (unpack(">IB", "\0"+self.file.read(4)) for block in xrange(1024)) + if extent_offset != 0 and ( sector >= extent_offset < (sector+nsectors))): + #move foward to end of intersect + sector = intersect_offset + intersect_len + found = False + break if found: break @@ -418,5 +417,4 @@ def write_chunk(self, x, z, nbt_file): #write timestamp self.file.seek(4096+4*(x+z*32)) timestamp = time.mktime(datetime.datetime.now().timetuple()) - self.file.write(pack(">I", (timestamp,)) - \ No newline at end of file + self.file.write(pack(">I", (timestamp,))) From 04e4286dc0b34240f458754b5fc00416ae96482e Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Tue, 15 Mar 2011 15:36:06 -0500 Subject: [PATCH 02/40] Sample code for generating a level.dat file --- generate_level_dat.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 generate_level_dat.py diff --git a/generate_level_dat.py b/generate_level_dat.py new file mode 100644 index 0000000..7339c2e --- /dev/null +++ b/generate_level_dat.py @@ -0,0 +1,34 @@ +# Create a file that can be used as a basic level.dat file with all required fields +# http://www.minecraftwiki.net/wiki/Alpha_Level_Format#level.dat_Format + +from nbt import * +import time +import random + +level = NBTFile() # Blank NBT +level.name = "Data" +level.tags.extend([ + TAG_Long(name="Time", value=1), + TAG_Long(name="LastPlayed", value=int(time.time())), + TAG_Int(name="SpawnX", value=0), + TAG_Int(name="SpawnY", value=2), + TAG_Int(name="SpawnZ", value=0), + TAG_Long(name="SizeOnDisk", value=0), + TAG_Long(name="RandomSeed", value=random.randrange(1,9999999999)), + TAG_Int(name="version", value=19132), + TAG_String(name="LevelName", value="Testing") +]) + +player = TAG_Compound() +player.name = "Player" +player.tags.extend([ + TAG_Int(name="Score", value=0), + TAG_Int(name="Dimension", value=0) +]) +inventory = TAG_Compound() +inventory.name = "Inventory" +player.tags.append(inventory) +level.tags.append(player) + +print level.pretty_tree() +#level.write_file("level.dat") \ No newline at end of file From b024755be263a86a1675d330f82cc8a7d42c4a9a Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Tue, 15 Mar 2011 15:37:07 -0500 Subject: [PATCH 03/40] Syntax fix --- nbt/nbt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nbt/nbt.py b/nbt/nbt.py index f534845..11a20ed 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -356,6 +356,9 @@ def __del__(self): if self.file: self.file.close() + def parse_header(self): + pass + @classmethod def getchunk(path, x, z): pass From 4836d5ed27262b8ffd9688c97c37fe53c05be1b1 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Tue, 15 Mar 2011 15:37:53 -0500 Subject: [PATCH 04/40] Get list of chunks that are in a region file --- nbt/nbt.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/nbt/nbt.py b/nbt/nbt.py index 11a20ed..5a63ed6 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -338,6 +338,17 @@ def write_file(self, filename=None, buffer=None, fileobj=None): self.file.close() ### WARNING! NOT EXTENSIVELY TESTED! ### +class Chunk(object): + def __init__(self, x, z, length): + self.x = x if x else 0 + self.z = z if z else 0 + self.length = length if length else 0 + + def toString(self): + return "("+str(self.x)+","+str(self.z)+"): "+str(self.length) + + + class RegionFile(object): """A convenience class for extracting NBT files from the new minecraft Region Format""" @@ -359,6 +370,19 @@ def __del__(self): def parse_header(self): pass + def get_chunks(self): + index = 0 + self.file.seek(index) + chunks = [] + while (index < 4096): + offset, length = unpack(">IB", "\0"+self.file.read(4)) + if offset: + x = (index/4) % 32 + z = int(index/4)/32 + chunks.append(Chunk(x,z,length)) + index += 4 + return chunks + @classmethod def getchunk(path, x, z): pass From 4683c9015ec2c01d9482559fb973e95da118193e Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Tue, 15 Mar 2011 15:38:20 -0500 Subject: [PATCH 05/40] Proper handling of getting a chunk out of a region file --- nbt/nbt.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/nbt/nbt.py b/nbt/nbt.py index 5a63ed6..f2a0b6b 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -1,5 +1,6 @@ from struct import pack, unpack, calcsize, error as StructError from gzip import GzipFile +import zlib from UserDict import DictMixin from StringIO import StringIO import os, io, math, time, datetime @@ -396,9 +397,21 @@ def get_chunk(self, x, z): block = 4*(x+z*32) self.file.seek(block) offset, length = unpack(">IB", "\0"+self.file.read(4)) + offset = offset * 1024*4 # offset is in 4KiB sectors if offset: self.file.seek(offset) - return NBTFile(fileobj=self.file) + length = unpack(">I", self.file.read(4)) + length = length[0] # For some reason, this is coming back as a tuple + compression = unpack(">B", self.file.read(1)) + compression = compression[0] + chunk = self.file.read(length-1) + if (compression == 2): + chunk = zlib.decompress(chunk) + else: + chunk = GzipFile(chunk) + + chunk = StringIO(chunk) + return NBTFile(buffer=chunk) else: return None From 225a7e6929a2e822c5d798cb160df925a0401859 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Wed, 16 Mar 2011 00:43:10 -0500 Subject: [PATCH 06/40] Use proper python __repr__ method --- nbt/nbt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbt/nbt.py b/nbt/nbt.py index f2a0b6b..abcf5e9 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -345,7 +345,7 @@ def __init__(self, x, z, length): self.z = z if z else 0 self.length = length if length else 0 - def toString(self): + def __repr__(self): return "("+str(self.x)+","+str(self.z)+"): "+str(self.length) From 0d5a80726f2459f53feabbf40eaf1f0faa9e551b Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Wed, 16 Mar 2011 00:44:07 -0500 Subject: [PATCH 07/40] If chunk is GZipped, it needs to be a fileobj. ZLib chunks need to be buffers. --- nbt/nbt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nbt/nbt.py b/nbt/nbt.py index abcf5e9..8524761 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -407,11 +407,11 @@ def get_chunk(self, x, z): chunk = self.file.read(length-1) if (compression == 2): chunk = zlib.decompress(chunk) + chunk = StringIO(chunk) + return NBTFile(buffer=chunk) else: chunk = GzipFile(chunk) - - chunk = StringIO(chunk) - return NBTFile(buffer=chunk) + return NBTFile(fileobj=chunk) else: return None From 372b41c638c79d23ffbdde147f0b1e6a0fe23d81 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Wed, 16 Mar 2011 14:16:48 -0500 Subject: [PATCH 08/40] Restructuring file layout --- MANIFEST.in | 2 +- nbt/bigtest.nbt => bigtest.nbt | Bin nbt/chunk.py | 58 +++++++++++++++ nbt/nbt.py | 126 +-------------------------------- nbt/region.py | 118 ++++++++++++++++++++++++++++++ nbt/tests.py => tests.py | 0 6 files changed, 179 insertions(+), 125 deletions(-) rename nbt/bigtest.nbt => bigtest.nbt (100%) create mode 100644 nbt/chunk.py create mode 100644 nbt/region.py rename nbt/tests.py => tests.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in index 6cfb7cd..a23f3d3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ include *.txt -include nbt\bigtest.nbt \ No newline at end of file +include bigtest.nbt \ No newline at end of file diff --git a/nbt/bigtest.nbt b/bigtest.nbt similarity index 100% rename from nbt/bigtest.nbt rename to bigtest.nbt diff --git a/nbt/chunk.py b/nbt/chunk.py new file mode 100644 index 0000000..f7312c2 --- /dev/null +++ b/nbt/chunk.py @@ -0,0 +1,58 @@ +""" Handle a single chunk of data (16x16x128 blocks) """ +class Chunk(object): + def __init__(self, x, z, length): + self.coords = x,z + self.length = length + + def __repr__(self): + return "("+str(self.coords[0])+","+str(self.coords[1])+"): "+str(self.length) + +def ByteToHex(byteStr): + return "".join(["%02X " % ord(x) for x in byteStr]).strip() + +""" Convenience class for dealing with a Block/data byte array """ +class BlockArray(object): + def __init__(self, blocksBytes, dataBytes): + self.blocksList = [ord(b) for b in blocksBytes] # A list of bytes + self.dataList = [ord(b) for b in dataBytes] + + # Get all data entries + def get_all_data(self): + bits = [] + for b in self.dataList: + bits.append((b >> 15) & 15) # Big end of the byte + bits.append(b & 15) # Little end of the byte + return bits + + # Get a given X,Y,Z + def get_block(self, x,y,z): + """ + Laid out like: + (0,0,0), (0,1,0), (0,2,0) ... (0,127,0), (0,0,1), (0,1,1), (0,2,1) ... (0,127,1), (0,0,2) ... (0,127,15), (1,0,0), (1,1,0) ... (15,127,15) + + blocks = [] + for x in xrange(15): + for z in xrange(15): + for y in xrange(127): + blocks.append(Block(x,y,z)) + """ + + offset = y + z*128 + x*128*16 + return self.blocksList[offset] + + # Get a given X,Y,Z + def get_data(self, x,y,z): + offset = y + z*128 + x*128*16 + #print "Offset: "+str(offset) + if (offset % 2 == 1): + # offset is odd + index = (offset-1)/2 + b = self.dataList[index] + #print "Byte: %02X" % b + return (b >>15) & 15 # Get big end of byte + else: + # offset is even + index = offset/2 + b = self.dataList[index] + #print "Byte: %02X" % b + return b & 15 \ No newline at end of file diff --git a/nbt/nbt.py b/nbt/nbt.py index 8524761..1f29390 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -2,8 +2,7 @@ from gzip import GzipFile import zlib from UserDict import DictMixin -from StringIO import StringIO -import os, io, math, time, datetime +import os, io TAG_END = 0 TAG_BYTE = 1 @@ -336,125 +335,4 @@ def write_file(self, filename=None, buffer=None, fileobj=None): if 'flush' in dir(self.file): self.file.flush() if self.filename and 'close' in dir(self.file): - self.file.close() - -### WARNING! NOT EXTENSIVELY TESTED! ### -class Chunk(object): - def __init__(self, x, z, length): - self.x = x if x else 0 - self.z = z if z else 0 - self.length = length if length else 0 - - def __repr__(self): - return "("+str(self.x)+","+str(self.z)+"): "+str(self.length) - - - -class RegionFile(object): - """A convenience class for extracting NBT files from the new minecraft Region Format""" - - def __init__(self, filename=None, fileobj=None): - if filename: - self.filename = filename - self.file = open(filename, 'rb') - if fileobj: - self.file = fileobj - self.chunks = [] - self.extents = None - if self.file: - self.parse_header() - - def __del__(self): - if self.file: - self.file.close() - - def parse_header(self): - pass - - def get_chunks(self): - index = 0 - self.file.seek(index) - chunks = [] - while (index < 4096): - offset, length = unpack(">IB", "\0"+self.file.read(4)) - if offset: - x = (index/4) % 32 - z = int(index/4)/32 - chunks.append(Chunk(x,z,length)) - index += 4 - return chunks - - @classmethod - def getchunk(path, x, z): - pass - - def get_timestamp(self, x, z): - self.file.seek(4096+4*(x+z*32)) - timestamp = unpack(">I",self.file.read(4)) - - def get_chunk(self, x, z): - #read metadata block - block = 4*(x+z*32) - self.file.seek(block) - offset, length = unpack(">IB", "\0"+self.file.read(4)) - offset = offset * 1024*4 # offset is in 4KiB sectors - if offset: - self.file.seek(offset) - length = unpack(">I", self.file.read(4)) - length = length[0] # For some reason, this is coming back as a tuple - compression = unpack(">B", self.file.read(1)) - compression = compression[0] - chunk = self.file.read(length-1) - if (compression == 2): - chunk = zlib.decompress(chunk) - chunk = StringIO(chunk) - return NBTFile(buffer=chunk) - else: - chunk = GzipFile(chunk) - return NBTFile(fileobj=chunk) - else: - return None - - def write_chunk(self, x, z, nbt_file): - """ A smart chunk writer that uses extents to trade off between fragmentation and cpu time""" - data = StringIO() - data.seek(0) - nbtfile.write_file(fileobj = data) #render to buffer - nsectors = int(math.ceil(data.len/4096)) - - #if it will fit back in it's orriginal slot: - self.file.seek(4*(x+z*32)) - offset, length = unpack(">IB", "\0"+self.file.read(4)) - if nsectors <= length: - sector = offset - - #traverse extents to find first-fit - else: - sector= 2 #start at sector 2, first sector after header - while 1: - #check if extent is used, else move foward in extent list by extent length - self.file.seek(0) - found = True - for intersect_offset, intersect_len in ( (extent_offset, extent_len) - for extent_offset, extent_len in (unpack(">IB", "\0"+self.file.read(4)) for block in xrange(1024)) - if extent_offset != 0 and ( sector >= extent_offset < (sector+nsectors))): - #move foward to end of intersect - sector = intersect_offset + intersect_len - found = False - break - if found: - break - - #write out chunk to region - self.file.seek(sector*4096) - self.file.write(pack(">I", (data.len,))) #length field - self.dile.write(data.getvalue()) #compressed data - - #seek to header record and write offset and length records - self.file.seek(4*(x+z*32)) - self.file.write(pack(">IB", (sector, length))[1:]) - - #write timestamp - self.file.seek(4096+4*(x+z*32)) - timestamp = time.mktime(datetime.datetime.now().timetuple()) - self.file.write(pack(">I", (timestamp,))) + self.file.close() \ No newline at end of file diff --git a/nbt/region.py b/nbt/region.py new file mode 100644 index 0000000..31306e8 --- /dev/null +++ b/nbt/region.py @@ -0,0 +1,118 @@ +from nbt import NBTFile +from chunk import Chunk +from struct import pack, unpack +from gzip import GzipFile +import zlib +from StringIO import StringIO +import math, time, datetime + +class RegionFile(object): + """ + A convenience class for extracting NBT files from the Minecraft Beta Region Format + """ + + def __init__(self, filename=None, fileobj=None): + if filename: + self.filename = filename + self.file = open(filename, 'rb') + if fileobj: + self.file = fileobj + self.chunks = [] + self.extents = None + if self.file: + self.parse_header() + + def __del__(self): + if self.file: + self.file.close() + + def parse_header(self): + pass + + def get_chunks(self): + index = 0 + self.file.seek(index) + chunks = [] + while (index < 4096): + offset, length = unpack(">IB", "\0"+self.file.read(4)) + if offset: + x = (index/4) % 32 + z = int(index/4)/32 + chunks.append(Chunk(x,z,length)) + index += 4 + return chunks + + @classmethod + def getchunk(path, x, z): + pass + + def get_timestamp(self, x, z): + self.file.seek(4096+4*(x+z*32)) + timestamp = unpack(">I",self.file.read(4)) + + def get_chunk(self, x, z): + #read metadata block + block = 4*(x+z*32) + self.file.seek(block) + offset, length = unpack(">IB", "\0"+self.file.read(4)) + offset = offset * 1024*4 # offset is in 4KiB sectors + if offset: + self.file.seek(offset) + length = unpack(">I", self.file.read(4)) + length = length[0] # For some reason, this is coming back as a tuple + compression = unpack(">B", self.file.read(1)) + compression = compression[0] + chunk = self.file.read(length-1) + if (compression == 2): + chunk = zlib.decompress(chunk) + chunk = StringIO(chunk) + return NBTFile(buffer=chunk) + else: + chunk = GzipFile(chunk) + return NBTFile(fileobj=chunk) + else: + return None + + def write_chunk(self, x, z, nbt_file): + """ A smart chunk writer that uses extents to trade off between fragmentation and cpu time""" + data = StringIO() + data.seek(0) + nbtfile.write_file(fileobj = data) #render to buffer + nsectors = int(math.ceil(data.len/4096)) + + #if it will fit back in it's orriginal slot: + self.file.seek(4*(x+z*32)) + offset, length = unpack(">IB", "\0"+self.file.read(4)) + if nsectors <= length: + sector = offset + + #traverse extents to find first-fit + else: + sector= 2 #start at sector 2, first sector after header + while 1: + #check if extent is used, else move foward in extent list by extent length + self.file.seek(0) + found = True + for intersect_offset, intersect_len in ( (extent_offset, extent_len) + for extent_offset, extent_len in (unpack(">IB", "\0"+self.file.read(4)) for block in xrange(1024)) + if extent_offset != 0 and ( sector >= extent_offset < (sector+nsectors))): + #move foward to end of intersect + sector = intersect_offset + intersect_len + found = False + break + if found: + break + + #write out chunk to region + self.file.seek(sector*4096) + self.file.write(pack(">I", (data.len,))) #length field + self.dile.write(data.getvalue()) #compressed data + + #seek to header record and write offset and length records + self.file.seek(4*(x+z*32)) + self.file.write(pack(">IB", (sector, length))[1:]) + + #write timestamp + self.file.seek(4096+4*(x+z*32)) + timestamp = time.mktime(datetime.datetime.now().timetuple()) + self.file.write(pack(">I", (timestamp,))) diff --git a/nbt/tests.py b/tests.py similarity index 100% rename from nbt/tests.py rename to tests.py From 321743ba01c86cfc023f0214e1f45cdd8f76a2ec Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Thu, 17 Mar 2011 15:22:45 -0500 Subject: [PATCH 09/40] Naming error --- nbt/region.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbt/region.py b/nbt/region.py index 31306e8..6ffdb40 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -77,7 +77,7 @@ def write_chunk(self, x, z, nbt_file): """ A smart chunk writer that uses extents to trade off between fragmentation and cpu time""" data = StringIO() data.seek(0) - nbtfile.write_file(fileobj = data) #render to buffer + nbt_file.write_file(fileobj = data) #render to buffer nsectors = int(math.ceil(data.len/4096)) #if it will fit back in it's orriginal slot: From 5758c881b74ad8791b54670173697b7eab8b921d Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Thu, 17 Mar 2011 15:23:13 -0500 Subject: [PATCH 10/40] zlib.decompress can handle zlib and gzip-compressed objects, methinks --- nbt/region.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index 6ffdb40..c56542f 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -63,13 +63,9 @@ def get_chunk(self, x, z): compression = unpack(">B", self.file.read(1)) compression = compression[0] chunk = self.file.read(length-1) - if (compression == 2): - chunk = zlib.decompress(chunk) - chunk = StringIO(chunk) - return NBTFile(buffer=chunk) - else: - chunk = GzipFile(chunk) - return NBTFile(fileobj=chunk) + chunk = zlib.decompress(chunk) + chunk = StringIO(chunk) + return NBTFile(buffer=chunk) else: return None From cc91b60812c4e01556e6fa2ff767b5c6182913ce Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Thu, 17 Mar 2011 15:23:36 -0500 Subject: [PATCH 11/40] Allow TAG_Byte_Array objects to set their name when created. --- nbt/nbt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbt/nbt.py b/nbt/nbt.py index 1f29390..43bcfd7 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -90,8 +90,8 @@ class TAG_Double(_TAG_Numeric): class TAG_Byte_Array(TAG): id = TAG_BYTE_ARRAY - def __init__(self, buffer=None): - super(TAG_Byte_Array, self).__init__() + def __init__(self, name=None, buffer=None): + super(TAG_Byte_Array, self).__init__(name=name) self.value = '' if buffer: self._parse_buffer(buffer) From 4338f601d08f16a0dc19acf25a0ea30094b2579a Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Sat, 19 Mar 2011 09:08:53 -0500 Subject: [PATCH 12/40] Reset filename, if exporting multiple ways --- nbt/nbt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nbt/nbt.py b/nbt/nbt.py index 43bcfd7..3e94a75 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -316,6 +316,7 @@ def parse_file(self, filename=None, buffer=None, fileobj=None): else: ValueError("need a file!") def write_file(self, filename=None, buffer=None, fileobj=None): + self.filename = None if buffer: self.file = buffer elif filename: From a63a8ff0f5d533fa37a69ef2910ab1d8f9eece2b Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Sat, 19 Mar 2011 09:09:25 -0500 Subject: [PATCH 13/40] Buffer needs to be writable --- nbt/nbt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbt/nbt.py b/nbt/nbt.py index 3e94a75..246292b 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -323,7 +323,7 @@ def write_file(self, filename=None, buffer=None, fileobj=None): self.filename = filename self.file = GzipFile(filename, "wb") elif fileobj: - self.file = GzipFile(fileobj=fileobj) + self.file = GzipFile(fileobj=fileobj, mode="wb") elif self.filename: self.file = GzipFile(self.filename, "wb") elif not self.file: From 36be9d1fd20574ec4c654670117afeee93e30456 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Sat, 19 Mar 2011 09:15:54 -0500 Subject: [PATCH 14/40] Need to be able to update if self.write_chunk is used --- nbt/region.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbt/region.py b/nbt/region.py index c56542f..8c05f95 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -14,7 +14,7 @@ class RegionFile(object): def __init__(self, filename=None, fileobj=None): if filename: self.filename = filename - self.file = open(filename, 'rb') + self.file = open(filename, 'r+b') if fileobj: self.file = fileobj self.chunks = [] From 3fbdc525bf2cdf8f78b5385ee36b27cb415d8658 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Sat, 19 Mar 2011 09:17:40 -0500 Subject: [PATCH 15/40] The two compression methods ARE different (undo 5da388be01fe137910dc807c66dbe7197467a729) Compression mode 1 (GZip: RFC1952) needs the GzipFile parser, while mode 2 (Zlib: RFC1950) needs the zlib.decompress() and zlib.compress() to parse. --- nbt/region.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index 8c05f95..91bad16 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -63,9 +63,13 @@ def get_chunk(self, x, z): compression = unpack(">B", self.file.read(1)) compression = compression[0] chunk = self.file.read(length-1) - chunk = zlib.decompress(chunk) - chunk = StringIO(chunk) - return NBTFile(buffer=chunk) + if (compression == 2): + chunk = zlib.decompress(chunk) + chunk = StringIO(chunk) + return NBTFile(buffer=chunk) #pass uncompressed + else: + chunk = StringIO(chunk) + return NBTFile(fileobj=chunk) #pass compressed; will be filtered through Gzip else: return None From 4c98321f6a1edc4f13c319b8c302349b7f279c67 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Sat, 19 Mar 2011 09:19:33 -0500 Subject: [PATCH 16/40] Use the "standard" Zlib compression rather than Gzip compression method. Also, deal with zero-length data strings to force chunks to take up at least one sector. --- nbt/region.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index 91bad16..a4315ca 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -76,11 +76,14 @@ def get_chunk(self, x, z): def write_chunk(self, x, z, nbt_file): """ A smart chunk writer that uses extents to trade off between fragmentation and cpu time""" data = StringIO() - data.seek(0) - nbt_file.write_file(fileobj = data) #render to buffer - nsectors = int(math.ceil(data.len/4096)) + nbt_file.write_file(buffer = data) #render to buffer; uncompressed - #if it will fit back in it's orriginal slot: + compressed = zlib.compress(data.getvalue()) #use zlib compression, rather than Gzip + data = StringIO(compressed) + + nsectors = int(math.ceil((data.len+0.001)/4096)) + + #if it will fit back in it's original slot: self.file.seek(4*(x+z*32)) offset, length = unpack(">IB", "\0"+self.file.read(4)) if nsectors <= length: From 68c0a1b51fd869e8f9eabc26bf3cfc609f725f8f Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Sat, 19 Mar 2011 09:21:20 -0500 Subject: [PATCH 17/40] If offset and length are zero, it means the chunk doesn't exist; so there's no prior size that it fit in. Just put it at the end of the file. --- nbt/region.py | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index a4315ca..e1b8354 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -86,26 +86,33 @@ def write_chunk(self, x, z, nbt_file): #if it will fit back in it's original slot: self.file.seek(4*(x+z*32)) offset, length = unpack(">IB", "\0"+self.file.read(4)) - if nsectors <= length: - sector = offset - - #traverse extents to find first-fit + if (offset == 0 and length == 0): + # This chunk hasn't been generated yet + # This chunk should just be appended to the end of the file + self.file.seek(0,2) # go to the end of the file + file_length = self.file.tell()-1 # current offset is file length + total_sectors = file_length/4096 + sector = total_sectors+1 else: - sector= 2 #start at sector 2, first sector after header - while 1: - #check if extent is used, else move foward in extent list by extent length - self.file.seek(0) - found = True - for intersect_offset, intersect_len in ( (extent_offset, extent_len) - for extent_offset, extent_len in (unpack(">IB", "\0"+self.file.read(4)) for block in xrange(1024)) - if extent_offset != 0 and ( sector >= extent_offset < (sector+nsectors))): - #move foward to end of intersect - sector = intersect_offset + intersect_len - found = False - break - if found: - break - + if nsectors <= length: + sector = offset + else: + #traverse extents to find first-fit + sector= 2 #start at sector 2, first sector after header + while 1: + #check if extent is used, else move foward in extent list by extent length + self.file.seek(0) + found = True + for intersect_offset, intersect_len in ( (extent_offset, extent_len) + for extent_offset, extent_len in (unpack(">IB", "\0"+self.file.read(4)) for block in xrange(1024)) + if extent_offset != 0 and ( sector >= extent_offset < (sector+nsectors))): + #move foward to end of intersect + sector = intersect_offset + intersect_len + found = False + break + if found: + break + #write out chunk to region self.file.seek(sector*4096) self.file.write(pack(">I", (data.len,))) #length field From 4059d3861fa976646c225a5d390c465299c41b91 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Sat, 19 Mar 2011 09:23:03 -0500 Subject: [PATCH 18/40] Don't need to pass a tuple to the pack function, the length field needs to be one more than it actually is, the compression field was missing, and the header length field was using the old chunk's size, rather than the new data's size. --- nbt/region.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index e1b8354..f87ca43 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -115,14 +115,15 @@ def write_chunk(self, x, z, nbt_file): #write out chunk to region self.file.seek(sector*4096) - self.file.write(pack(">I", (data.len,))) #length field - self.dile.write(data.getvalue()) #compressed data + self.file.write(pack(">I", data.len+1)) #length field + self.file.write(pack(">B", 2)) #compression field + self.file.write(data.getvalue()) #compressed data #seek to header record and write offset and length records self.file.seek(4*(x+z*32)) - self.file.write(pack(">IB", (sector, length))[1:]) + self.file.write(pack(">IB", sector, nsectors)[1:]) #write timestamp self.file.seek(4096+4*(x+z*32)) timestamp = time.mktime(datetime.datetime.now().timetuple()) - self.file.write(pack(">I", (timestamp,))) + self.file.write(pack(">I", timestamp)) From c67321ae62a477b8db7c15f4854449229b457850 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Sun, 20 Mar 2011 16:46:30 -0500 Subject: [PATCH 19/40] TAG_Compounds default to an empty name --- nbt/nbt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nbt/nbt.py b/nbt/nbt.py index 246292b..6ac1a04 100644 --- a/nbt/nbt.py +++ b/nbt/nbt.py @@ -184,6 +184,7 @@ class TAG_Compound(TAG, DictMixin): def __init__(self, buffer=None): super(TAG_Compound, self).__init__() self.tags = [] + self.name = "" if buffer: self._parse_buffer(buffer) From 9cf59e048a246faab95c8283ba5eb3f0e07ffec8 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Sun, 20 Mar 2011 16:47:06 -0500 Subject: [PATCH 20/40] If adding to the end of a file, Minecraft needs there to be zero-padding at the end to make it eactly 4096 for each sector. --- nbt/region.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nbt/region.py b/nbt/region.py index f87ca43..148c862 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -86,6 +86,7 @@ def write_chunk(self, x, z, nbt_file): #if it will fit back in it's original slot: self.file.seek(4*(x+z*32)) offset, length = unpack(">IB", "\0"+self.file.read(4)) + pad_end = False if (offset == 0 and length == 0): # This chunk hasn't been generated yet # This chunk should just be appended to the end of the file @@ -93,6 +94,7 @@ def write_chunk(self, x, z, nbt_file): file_length = self.file.tell()-1 # current offset is file length total_sectors = file_length/4096 sector = total_sectors+1 + pad_end = True else: if nsectors <= length: sector = offset @@ -118,6 +120,10 @@ def write_chunk(self, x, z, nbt_file): self.file.write(pack(">I", data.len+1)) #length field self.file.write(pack(">B", 2)) #compression field self.file.write(data.getvalue()) #compressed data + if pad_end: + # Write zeros up to the end of the chunk + self.file.seek((sector+nsectors)*4096-1) + self.file.write(chr(0)) #seek to header record and write offset and length records self.file.seek(4*(x+z*32)) From cf536a868f0f6aaba735f8e7039b2eead0f2d11f Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Sun, 3 Apr 2011 23:22:55 +0200 Subject: [PATCH 21/40] Basic handling of corrupted regions and chunks. --- nbt/region.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/nbt/region.py b/nbt/region.py index 148c862..a351937 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -5,6 +5,18 @@ import zlib from StringIO import StringIO import math, time, datetime +from os.path import getsize + +class RegionHeaderError(Exception): + """Error in the header of the region file for a given chunk""" + def __init__(self, msg): + self.msg = msg + +class ChunkDataError(Exception): + """Error in the data of a chunk, included the bytes of lenght and byte version""" + def __init__(self, msg): + self.msg = msg + class RegionFile(object): """ @@ -56,10 +68,21 @@ def get_chunk(self, x, z): self.file.seek(block) offset, length = unpack(">IB", "\0"+self.file.read(4)) offset = offset * 1024*4 # offset is in 4KiB sectors + if offset >= getsize(self.filename) - 1024*4: # mininmun chunk size = 1 sector + raise RegionHeaderError('The offset of the chunk is outside the file') + if offset: self.file.seek(offset) length = unpack(">I", self.file.read(4)) length = length[0] # For some reason, this is coming back as a tuple + if length == 0: # no chunk can be 0 length! + raise ChunkDataError('The length of the chunk is 0') + + if length > 32768 + 16384 + 16384 + 16384 + 256 + 1024: + # aprox size of an uncompressed chunk: blocks + data + skylight + block light + heightmap + entities(~1024?) + # also a chunk can't be bigger than 1MB + raise ChunkDataError('The length of the chunk is too big') + compression = unpack(">B", self.file.read(1)) compression = compression[0] chunk = self.file.read(length-1) From b594130695602f38efb58cc5d7403470f9bedf57 Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Sat, 2 Apr 2011 08:14:47 +0800 Subject: [PATCH 22/40] Add source of information. Add method .unlink that deletes the header of a chunk. This works as a deleted chunk, but leaving the data in the region file. --- nbt/region.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nbt/region.py b/nbt/region.py index 148c862..b8577c0 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -1,3 +1,8 @@ +# +# For more info of the region file format look: +# http://www.minecraftforum.net/viewtopic.php?f=25&t=120160 +# + from nbt import NBTFile from chunk import Chunk from struct import pack, unpack @@ -133,3 +138,12 @@ def write_chunk(self, x, z, nbt_file): self.file.seek(4096+4*(x+z*32)) timestamp = time.mktime(datetime.datetime.now().timetuple()) self.file.write(pack(">I", timestamp)) + + + def unlink_chunk(self, x, z): + """ Removes a chunk from the header of the region file (write zeros in the offset of the chunk). + Using only this method leaves the chunk data intact, fragmenting the region file (unconfirmed). + This is an start to a better function remove_chunk""" + + self.file.seek(4*(x+z*32)) + self.file.write(pack(">IB", 0, 0)[1:]) From 3cde4ed321f78bdbd90aaf479b467a4cb0f7b08e Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Mon, 4 Apr 2011 20:05:55 +0800 Subject: [PATCH 23/40] Better web for looking information. --- nbt/region.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbt/region.py b/nbt/region.py index b8577c0..1fc9444 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -1,6 +1,6 @@ # # For more info of the region file format look: -# http://www.minecraftforum.net/viewtopic.php?f=25&t=120160 +# http://www.minecraftwiki.net/wiki/Beta_Level_Format # from nbt import NBTFile From 33a91fa603c00be50f2e5207b7ae68be655e81ef Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Wed, 6 Apr 2011 11:57:23 -0500 Subject: [PATCH 24/40] Simplify timestamp creation There's no need to use three methods across two modules just to get a timestamp; use just the `time` module to simplify, and ensure its an integer --- nbt/region.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index 1fc9444..09d494a 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -9,7 +9,7 @@ from gzip import GzipFile import zlib from StringIO import StringIO -import math, time, datetime +import math, time class RegionFile(object): """ @@ -136,7 +136,7 @@ def write_chunk(self, x, z, nbt_file): #write timestamp self.file.seek(4096+4*(x+z*32)) - timestamp = time.mktime(datetime.datetime.now().timetuple()) + timestamp = int(time.time()) self.file.write(pack(">I", timestamp)) From 676666f1b2c282075eb08b62e7b4922f639f4565 Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Wed, 6 Apr 2011 23:12:52 +0200 Subject: [PATCH 25/40] Code parser_header for region files and needed changes in get_chunk. --- nbt/region.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index a351937..28920e8 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -13,7 +13,7 @@ def __init__(self, msg): self.msg = msg class ChunkDataError(Exception): - """Error in the data of a chunk, included the bytes of lenght and byte version""" + """Error in the data of a chunk, included the bytes of length and byte version""" def __init__(self, msg): self.msg = msg @@ -30,6 +30,7 @@ def __init__(self, filename=None, fileobj=None): if fileobj: self.file = fileobj self.chunks = [] + self.header = {} self.extents = None if self.file: self.parse_header() @@ -39,8 +40,22 @@ def __del__(self): self.file.close() def parse_header(self): - pass - + for index in range(0,4096,4): + self.file.seek(index) + offset, length = unpack(">IB", "\0"+self.file.read(4)) + x = (index/4) % 32 + z = int(index/4)/32 + if (offset + length)*4 > getsize(self.filename): # offset outside of file + status = -1 + + elif offset == 0: # no created yet + status = 1 + + else: + status = 0 # everything ok + + self.header[x,z] = (offset, length, status) + def get_chunks(self): index = 0 self.file.seek(index) @@ -64,25 +79,20 @@ def get_timestamp(self, x, z): def get_chunk(self, x, z): #read metadata block - block = 4*(x+z*32) - self.file.seek(block) - offset, length = unpack(">IB", "\0"+self.file.read(4)) - offset = offset * 1024*4 # offset is in 4KiB sectors - if offset >= getsize(self.filename) - 1024*4: # mininmun chunk size = 1 sector - raise RegionHeaderError('The offset of the chunk is outside the file') + offset, length, status = self.header[x, z] + if status == 1: + return None + + elif status == -1: + raise RegionHeaderError('The chunk is partially/completely outside the file') - if offset: - self.file.seek(offset) + elif status == 0: + self.file.seek(offset*4*1024) # offset comes in blocks of 4096 bytes length = unpack(">I", self.file.read(4)) length = length[0] # For some reason, this is coming back as a tuple if length == 0: # no chunk can be 0 length! raise ChunkDataError('The length of the chunk is 0') - if length > 32768 + 16384 + 16384 + 16384 + 256 + 1024: - # aprox size of an uncompressed chunk: blocks + data + skylight + block light + heightmap + entities(~1024?) - # also a chunk can't be bigger than 1MB - raise ChunkDataError('The length of the chunk is too big') - compression = unpack(">B", self.file.read(1)) compression = compression[0] chunk = self.file.read(length-1) From c028d8383bf75b15a341a87e413e1189e04dd4a0 Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Thu, 7 Apr 2011 00:19:29 +0200 Subject: [PATCH 26/40] Add parse_chunk_headers and needed stuff in __init__. Also add a new error number -2, means chunk inside the header of the region file. Add a new exception ChunkHeaderError for chunk header errors. --- nbt/region.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index 28920e8..0d9fa07 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -12,6 +12,11 @@ class RegionHeaderError(Exception): def __init__(self, msg): self.msg = msg +class ChunkHeaderError(Exception): + """Error in the header of a chunk""" + def __init__(self, msg): + self.msg = msg + class ChunkDataError(Exception): """Error in the data of a chunk, included the bytes of length and byte version""" def __init__(self, msg): @@ -31,21 +36,39 @@ def __init__(self, filename=None, fileobj=None): self.file = fileobj self.chunks = [] self.header = {} + self.chunk_headers = {} self.extents = None if self.file: + self.size = getsize(self.filename) self.parse_header() + self.parse_chunk_headers() def __del__(self): if self.file: self.file.close() def parse_header(self): + """ + Reads the region header and stores: offset, length and status. + + Status is a number representing: + -2 = Error, chunk inside the region file of the region file + -1 = Error, chunk partially/completely outside of file + 0 = Ok + 1 = Chunk non-existant yet + """ for index in range(0,4096,4): self.file.seek(index) offset, length = unpack(">IB", "\0"+self.file.read(4)) + self.file.seek(index + 4096) + timestamp = unpack(">I", self.file.read(4)) x = (index/4) % 32 z = int(index/4)/32 - if (offset + length)*4 > getsize(self.filename): # offset outside of file + + if offset < 2: # chunk inside the header of the region file + status = -2 + + elif (offset + length)*4 > self.size: # chunk outside of file status = -1 elif offset == 0: # no created yet @@ -54,8 +77,50 @@ def parse_header(self): else: status = 0 # everything ok - self.header[x,z] = (offset, length, status) + self.header[x,z] = (offset, length, timestamp, status) + + def parse_chunk_headers(self): + for x in range(32): + for z in range(32): + offset, region_header_length, timestamp, status = self.header[x,z] + + if status == 1: # chunk not created yet + length = None + compression = None + status = None + if status == 0: # there is a chunk! + self.file.seek(offset*4096) # offset comes in blocks of 4096 bytes + length = unpack(">I", self.file.read(4)) + length = length[0] # For some reason, this is coming back as a tuple + + compression = unpack(">B",self.file.read(1)) + # TODO TODO TODO check if the region_file_length and the chunk header length are compatible + status = 0 + + if status == -1: # error, chunk partially/completely outside the file + if offset*4096 + 5 < self.size: # if possible read it, just in case it's useful + self.file.seek(offset*4096) # offset comes in blocks of 4096 bytes + length = unpack(">I", self.file.read(4)) + length = length[0] # For some reason, this is coming back as a tuple + compression = unpack(">B",self.file.read(1)) + + else: + length = None + compression = None + status = -1 + + if status == -2: # error, chunk in the header of the region file + length = None + compression = None + status = -2 + + self.chunk_headers[x, z] = (length, compression, status) + + + def locate_free_space(self): + pass + def get_chunks(self): index = 0 self.file.seek(index) @@ -79,7 +144,7 @@ def get_timestamp(self, x, z): def get_chunk(self, x, z): #read metadata block - offset, length, status = self.header[x, z] + offset, length, timestamp, status = self.header[x, z] if status == 1: return None From e7457dfbdea7a99214090e9e9f67ee0c36d2a632 Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Thu, 7 Apr 2011 10:46:45 +0200 Subject: [PATCH 27/40] Changes in get_chunk to use the new parser_chunk_headers --- nbt/region.py | 69 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index 0d9fa07..e3d1c18 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -21,7 +21,7 @@ class ChunkDataError(Exception): """Error in the data of a chunk, included the bytes of length and byte version""" def __init__(self, msg): self.msg = msg - + class RegionFile(object): """ @@ -64,8 +64,7 @@ def parse_header(self): timestamp = unpack(">I", self.file.read(4)) x = (index/4) % 32 z = int(index/4)/32 - - if offset < 2: # chunk inside the header of the region file + if offset < 2 and offset != 0: # chunk inside the header of the region file status = -2 elif (offset + length)*4 > self.size: # chunk outside of file @@ -89,28 +88,33 @@ def parse_chunk_headers(self): compression = None status = None - if status == 0: # there is a chunk! + elif status == 0: # there is a chunk! self.file.seek(offset*4096) # offset comes in blocks of 4096 bytes length = unpack(">I", self.file.read(4)) - length = length[0] # For some reason, this is coming back as a tuple - + length = length[0] # unpack always returns a tuple, even unpacking one element compression = unpack(">B",self.file.read(1)) + compression = compression[0] # TODO TODO TODO check if the region_file_length and the chunk header length are compatible - status = 0 + if length == 0: # chunk can't be zero length + status = -3 + + else: + status = 0 - if status == -1: # error, chunk partially/completely outside the file + elif status == -1: # error, chunk partially/completely outside the file if offset*4096 + 5 < self.size: # if possible read it, just in case it's useful self.file.seek(offset*4096) # offset comes in blocks of 4096 bytes length = unpack(">I", self.file.read(4)) - length = length[0] # For some reason, this is coming back as a tuple + length = length[0] # unpack always returns a tuple, even unpacking one element compression = unpack(">B",self.file.read(1)) - + compression = compression[0] + else: length = None compression = None status = -1 - if status == -2: # error, chunk in the header of the region file + elif status == -2: # error, chunk in the header of the region file length = None compression = None status = -2 @@ -144,30 +148,43 @@ def get_timestamp(self, x, z): def get_chunk(self, x, z): #read metadata block - offset, length, timestamp, status = self.header[x, z] - if status == 1: + offset, length, timestamp, region_header_status = self.header[x, z] + if region_header_status == 1: return None + + elif region_header_status == -2: + raise RegionHeaderError('The chunk is in the region header') - elif status == -1: + elif region_header_status == -1: raise RegionHeaderError('The chunk is partially/completely outside the file') - elif status == 0: - self.file.seek(offset*4*1024) # offset comes in blocks of 4096 bytes - length = unpack(">I", self.file.read(4)) - length = length[0] # For some reason, this is coming back as a tuple - if length == 0: # no chunk can be 0 length! - raise ChunkDataError('The length of the chunk is 0') + elif region_header_status == 0: + length, compression, chunk_header_status = self.chunk_headers[x, z] + if chunk_header_status == -3: # no chunk can be 0 length! + raise ChunkHeaderError('The length of the chunk is 0') - compression = unpack(">B", self.file.read(1)) - compression = compression[0] + self.file.seek(offset*4*1024 + 5) # offset comes in blocks of 4096 bytes + length bytes + compression byte chunk = self.file.read(length-1) + if (compression == 2): - chunk = zlib.decompress(chunk) + try: + chunk = zlib.decompress(chunk) + except Exception, e: + raise ChunkDataError(str(e)) + chunk = StringIO(chunk) - return NBTFile(buffer=chunk) #pass uncompressed - else: + return NBTFile(buffer=chunk) # pass uncompressed + + elif (compression == 1): chunk = StringIO(chunk) - return NBTFile(fileobj=chunk) #pass compressed; will be filtered through Gzip + try: + return NBTFile(fileobj=chunk) # pass compressed; will be filtered through Gzip + except Exception, e: + raise ChunkDataError(str(e)) + + else: + raise ChunkDataError('Unknown chunk compression') + else: return None From f3df1c748556953858784030092200b0f76ca83e Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Thu, 7 Apr 2011 15:17:34 +0200 Subject: [PATCH 28/40] Update some comments, partial update of write_chunk to use self.header --- nbt/region.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index e3d1c18..1e6bb9b 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -89,7 +89,7 @@ def parse_chunk_headers(self): status = None elif status == 0: # there is a chunk! - self.file.seek(offset*4096) # offset comes in blocks of 4096 bytes + self.file.seek(offset*4096) # offset comes in sectors of 4096 bytes length = unpack(">I", self.file.read(4)) length = length[0] # unpack always returns a tuple, even unpacking one element compression = unpack(">B",self.file.read(1)) @@ -103,7 +103,7 @@ def parse_chunk_headers(self): elif status == -1: # error, chunk partially/completely outside the file if offset*4096 + 5 < self.size: # if possible read it, just in case it's useful - self.file.seek(offset*4096) # offset comes in blocks of 4096 bytes + self.file.seek(offset*4096) # offset comes in sectors of 4096 bytes length = unpack(">I", self.file.read(4)) length = length[0] # unpack always returns a tuple, even unpacking one element compression = unpack(">B",self.file.read(1)) @@ -163,7 +163,7 @@ def get_chunk(self, x, z): if chunk_header_status == -3: # no chunk can be 0 length! raise ChunkHeaderError('The length of the chunk is 0') - self.file.seek(offset*4*1024 + 5) # offset comes in blocks of 4096 bytes + length bytes + compression byte + self.file.seek(offset*4*1024 + 5) # offset comes in sectors of 4096 bytes + length bytes + compression byte chunk = self.file.read(length-1) if (compression == 2): @@ -199,18 +199,18 @@ def write_chunk(self, x, z, nbt_file): nsectors = int(math.ceil((data.len+0.001)/4096)) #if it will fit back in it's original slot: - self.file.seek(4*(x+z*32)) - offset, length = unpack(">IB", "\0"+self.file.read(4)) + offset, length, timestamp, status = self.header[x, z] pad_end = False - if (offset == 0 and length == 0): - # This chunk hasn't been generated yet + if status in (1,-1,-2): # don't trust bad headers, write at the end. + # This chunk hasn't been generated yet, or the header is wrong # This chunk should just be appended to the end of the file self.file.seek(0,2) # go to the end of the file file_length = self.file.tell()-1 # current offset is file length total_sectors = file_length/4096 sector = total_sectors+1 pad_end = True - else: + elif status == 0: + # TODO TODO TODO Check if chunk_status says that the lengths are incompatible (status = -3) if nsectors <= length: sector = offset else: @@ -218,6 +218,7 @@ def write_chunk(self, x, z, nbt_file): sector= 2 #start at sector 2, first sector after header while 1: #check if extent is used, else move foward in extent list by extent length + # leave this like this or update to use self.header? self.file.seek(0) found = True for intersect_offset, intersect_len in ( (extent_offset, extent_len) From 28693f4d6dbe2db246471960a969d5e59efd76ea Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Thu, 7 Apr 2011 14:55:25 -0500 Subject: [PATCH 29/40] Update Chunk object with changes from mapbuilder branch --- nbt/chunk.py | 283 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 260 insertions(+), 23 deletions(-) diff --git a/nbt/chunk.py b/nbt/chunk.py index f7312c2..af49c40 100644 --- a/nbt/chunk.py +++ b/nbt/chunk.py @@ -1,31 +1,219 @@ """ Handle a single chunk of data (16x16x128 blocks) """ +from StringIO import StringIO +from struct import pack, unpack +import array, math + +try: + import Image + PIL_enabled = True +except ImportError: + PIL_enabled = False + class Chunk(object): - def __init__(self, x, z, length): - self.coords = x,z - self.length = length - - def __repr__(self): - return "("+str(self.coords[0])+","+str(self.coords[1])+"): "+str(self.length) + def __init__(self, nbt): + chunk_data = nbt['Level'] + self.coords = chunk_data['xPos'],chunk_data['zPos'] + self.blocks = BlockArray(chunk_data['Blocks'].value, chunk_data['Data'].value) + + def get_heightmap_image(self, buffer=False, gmin=False, gmax=False): + if (not PIL_enabled): return false + points = self.blocks.generate_heightmap(buffer, True) + # Normalize the points + hmin = min(points) if (gmin == False) else gmin # Allow setting the min/max explicitly, in case this is part of a bigger map + hmax = max(points) if (gmax == False) else gmax + hdelta = hmax-hmin+0.0 + pixels = "" + for y in range(16): + for x in range(16): + # pix X => mc -Z + # pix Y => mc X + offset = (15-x)*16+y + height = int((points[offset]-hmin)/hdelta*255) + if (height < 0): height = 0 + if (height > 255): height = 255 + pixels += pack(">B", height) + im = Image.fromstring('L', (16,16), pixels) + return im -def ByteToHex(byteStr): - return "".join(["%02X " % ord(x) for x in byteStr]).strip() + def get_map(self): + # Show an image of the chunk from above + if (not PIL_enabled): return false + pixels = "" + block_colors = { + 0: {'h':0, 's':0, 'l':0}, # Air + 1: {'h':0, 's':0, 'l':32}, # Stone + 2: {'h':94, 's':42, 'l':32}, # Grass + 3: {'h':27, 's':51, 'l':15}, # Dirt + 4: {'h':0, 's':0, 'l':25}, # Cobblestone + 8: {'h':228, 's':50, 'l':23}, # Water + 9: {'h':228, 's':50, 'l':23}, # Water + 10: {'h':16, 's':100, 'l':48}, # Lava + 11: {'h':16, 's':100, 'l':48}, # Lava + 12: {'h':53, 's':22, 'l':58}, # Sand + 13: {'h':21, 's':18, 'l':20}, # Gravel + 17: {'h':35, 's':93, 'l':15}, # Wood + 18: {'h':114, 's':64, 'l':22}, # Leaves + 24: {'h':48, 's':31, 'l':40}, # Sandstone + 37: {'h':60, 's':100, 'l':60}, # Yellow Flower + 38: {'h':0, 's':100, 'l':50}, # Red Flower + 50: {'h':60, 's':100, 'l':50}, # Torch + 51: {'h':55, 's':100, 'l':50}, # Fire + 59: {'h':123, 's':60, 'l':50}, # Crops + 60: {'h':35, 's':93, 'l':15}, # Farmland + 78: {'h':240, 's':10, 'l':85}, # Snow + 79: {'h':240, 's':10, 'l':95}, # Ice + 81: {'h':126, 's':61, 'l':20}, # Cacti + 82: {'h':7, 's':62, 'l':23}, # Clay + 83: {'h':123, 's':70, 'l':50}, # Sugarcane + 86: {'h':24, 's':100, 'l':45}, # Pumpkin + 91: {'h':24, 's':100, 'l':45}, # Jack-o-lantern + } + for x in range(16): + for z in range(15,-1,-1): + # Find the highest block in this column + ground_height = 127 + tints = [] + for y in range(127,-1,-1): + block_id = self.blocks.get_block(x,y,z) + block_data = self.blocks.get_data(x,y,z) + if (block_id == 8 or block_id == 9): + tints.append({'h':228, 's':50, 'l':23}) # Water + elif (block_id == 18): + if (block_data == 1): + tints.append({'h':114, 's':64, 'l':22}) # Redwood Leaves + elif (block_data == 2): + tints.append({'h':93, 's':39, 'l':10}) # Birch Leaves + else: + tints.append({'h':114, 's':64, 'l':22}) # Normal Leaves + elif (block_id == 79): + tints.append({'h':240, 's':5, 'l':95}) # Ice + elif (block_id == 51): + tints.append({'h':55, 's':100, 'l':50}) # Fire + elif (block_id != 0 or y == 0): + # Here is ground level + ground_height = y + break + + color = block_colors[block_id] if (block_id in block_colors) else {'h':0, 's':0, 'l':100} + height_shift = (ground_height-64)*0.25 + + final_color = {'h':color['h'], 's':color['s'], 'l':color['l']+height_shift} + if final_color['l'] > 100: final_color['l'] = 100 + if final_color['l'] < 0: final_color['l'] = 0 + + # Apply tints from translucent blocks + for tint in reversed(tints): + final_color = hsl_slide(final_color, tint, 0.4) + + rgb = hsl2rgb(final_color['h'], final_color['s'], final_color['l']) + + pixels += pack("BBB", rgb[0], rgb[1], rgb[2]) + im = Image.fromstring('RGB', (16,16), pixels) + return im + + def __repr__(self): + return "("+str(self.coords[0])+","+str(self.coords[1])+")" """ Convenience class for dealing with a Block/data byte array """ class BlockArray(object): - def __init__(self, blocksBytes, dataBytes): - self.blocksList = [ord(b) for b in blocksBytes] # A list of bytes - self.dataList = [ord(b) for b in dataBytes] + def __init__(self, blocksBytes=None, dataBytes=None): + if (blocksBytes != None): + self.blocksList = [ord(b) for b in blocksBytes] # A list of bytes + else: + self.blocksList = [0]*32768 # Create an empty block list (32768 entries of zero (air)) + + if (dataBytes != None): + self.dataList = [ord(b) for b in dataBytes] + else: + self.dataList = [0]*32768 # Create an empty data list (32768 entries of zero) # Get all data entries def get_all_data(self): bits = [] for b in self.dataList: - bits.append((b >> 15) & 15) # Big end of the byte + bits.append((b >> 4) & 15) # Big end of the byte bits.append(b & 15) # Little end of the byte return bits - - # Get a given X,Y,Z - def get_block(self, x,y,z): + + def get_blocks_struct(self): + cur_x = 0 + cur_y = 0 + cur_z = 0 + blocks = {} + for block_id in self.blocksList: + blocks[(cur_x,cur_y,cur_z)] = block_id + cur_y += 1 + if (cur_y > 127): + cur_y = 0 + cur_z += 1 + if (cur_z > 15): + cur_z = 0 + cur_x += 1 + return blocks + + # Give blockList back as a byte array + def get_blocks_byte_array(self, buffer=False): + if buffer: + length = len(self.blocksList) + return StringIO(pack(">i", length)+self.get_blocks_byte_array()) + else: + return array.array('B', self.blocksList).tostring() + + def get_data_byte_array(self, buffer=False): + if buffer: + length = len(self.dataList)/2 + return StringIO(pack(">i", length)+self.get_data_byte_array()) + else: + return array.array('B', self.dataList).tostring() + + def generate_heightmap(self, buffer=False, as_array=False): + if buffer: + return StringIO(pack(">i", 256)+self.generate_heightmap()) # Length + Heightmap, ready for insertion into Chunk NBT + else: + bytes = [] + for z in range(16): + for x in range(16): + for y in range(127, -1, -1): + offset = y + z*128 + x*128*16 + if (self.blocksList[offset] != 0 or y == 0): + bytes.append(y+1) + break + if (as_array): + return bytes + else: + return array.array('B', bytes).tostring() + + def set_blocks(self, list=None, dict=None, fill_air=False): + if list: + # Inputting a list like self.blocksList + self.blocksList = list + elif dict: + # Inputting a dictionary like result of self.get_blocks_struct() + list = [] + for x in range(16): + for z in range(16): + for y in range(128): + coord = x,y,z + offset = y + z*128 + x*128*16 + if (coord in dict): + list.append(dict[coord]) + else: + if (self.blocksList[offset] and not fill_air): + list.append(self.blocksList[offset]) + else: + list.append(0) # Air + self.blocksList = list + else: + # None of the above... + return False + return True + + def set_block(self, x,y,z, id): + offset = y + z*128 + x*128*16 + self.blocksList[offset] = id + + # Get a given X,Y,Z or a tuple of three coordinates + def get_block(self, x,y,z, coord=False): """ Laid out like: (0,0,0), (0,1,0), (0,2,0) ... (0,127,0), (0,0,1), (0,1,1), (0,2,1) ... (0,127,1), (0,0,2) ... (0,127,15), (1,0,0), (1,1,0) ... (15,127,15) @@ -37,22 +225,71 @@ def get_block(self, x,y,z): blocks.append(Block(x,y,z)) """ - offset = y + z*128 + x*128*16 + offset = y + z*128 + x*128*16 if (coord == False) else coord[1] + coord[2]*128 + coord[0]*128*16 return self.blocksList[offset] - # Get a given X,Y,Z - def get_data(self, x,y,z): - offset = y + z*128 + x*128*16 - #print "Offset: "+str(offset) + # Get a given X,Y,Z or a tuple of three coordinates + def get_data(self, x,y,z, coord=False): + offset = y + z*128 + x*128*16 if (coord == False) else coord[1] + coord[2]*128 + coord[0]*128*16 if (offset % 2 == 1): # offset is odd index = (offset-1)/2 b = self.dataList[index] - #print "Byte: %02X" % b return (b >>15) & 15 # Get big end of byte else: # offset is even index = offset/2 b = self.dataList[index] - #print "Byte: %02X" % b - return b & 15 \ No newline at end of file + return b & 15 + +## Color functions for map generation ## + +# Hue given in degrees, +# saturation and lightness given either in range 0-1 or 0-100 and returned in kind +def hsl_slide(hsl1, hsl2, ratio): + if (abs(hsl2['h'] - hsl1['h']) > 180): + if (hsl1['h'] > hsl2['h']): + hsl1['h'] -= 360 + else: + hsl1['h'] += 360 + + # Find location of two colors on the H/S color circle + p1x = math.cos(math.radians(hsl1['h']))*hsl1['s'] + p1y = math.sin(math.radians(hsl1['h']))*hsl1['s'] + p2x = math.cos(math.radians(hsl2['h']))*hsl2['s'] + p2y = math.sin(math.radians(hsl2['h']))*hsl2['s'] + + # Slide part of the way from tint to base color + avg_x = p1x + ratio*(p2x-p1x) + avg_y = p1y + ratio*(p2y-p1y) + avg_h = math.atan(avg_y/avg_x) + avg_s = avg_y/math.sin(avg_h) + avg_l = hsl1['l'] + ratio*(hsl2['l']-hsl1['l']) + avg_h = math.degrees(avg_h) + + #print 'tint:',tint, 'base:',final_color, 'avg:',avg_h,avg_s,avg_l + return {'h':avg_h, 's':avg_s, 'l':avg_l} + + +# From http://www.easyrgb.com/index.php?X=MATH&H=19#text19 +def hsl2rgb(H,S,L): + H = H/360.0 + S = S/100.0 # Turn into a percentage + L = L/100.0 + if (S == 0): + return (int(L*255), int(L*255), int(L*255)) + var_2 = L * (1+S) if (L < 0.5) else (L+S) - (S*L) + var_1 = 2*L - var_2 + + def hue2rgb(v1, v2, vH): + if (vH < 0): vH += 1 + if (vH > 1): vH -= 1 + if ((6*vH)<1): return v1 + (v2-v1)*6*vH + if ((2*vH)<1): return v2 + if ((3*vH)<2): return v1 + (v2-v1)*(2/3.0-vH)*6 + return v1 + + R = int(255*hue2rgb(var_1, var_2, H + (1.0/3))) + G = int(255*hue2rgb(var_1, var_2, H)) + B = int(255*hue2rgb(var_1, var_2, H - (1.0/3))) + return (R,G,B) \ No newline at end of file From d6ffde07844ac1c0e91e99e30ca586a98767adb3 Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Fri, 8 Apr 2011 09:03:50 +0200 Subject: [PATCH 30/40] Add unlink function to region.py (disappeared at some point...) --- nbt/region.py | 68 ++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index e029a88..28eb8c2 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -48,6 +48,19 @@ def __init__(self, filename=None, fileobj=None): self.parse_header() self.parse_chunk_headers() + # Status is a number representing: + # -3 = Error, chunk header has a 0 length + # -2 = Error, chunk inside the header of the region file + # -1 = Error, chunk partially/completely outside of file + # 0 = Ok + # 1 = Chunk non-existant yet + + self.ChunkZeroLength = -3 + self.ChunkInHeader = -2 + self.ChunkOutOfFile = -1 + self.ChunkOK = 0 + self.ChunkNotCreated = 1 + def __del__(self): if self.file: self.file.close() @@ -56,11 +69,6 @@ def parse_header(self): """ Reads the region header and stores: offset, length and status. - Status is a number representing: - -2 = Error, chunk inside the region file of the region file - -1 = Error, chunk partially/completely outside of file - 0 = Ok - 1 = Chunk non-existant yet """ for index in range(0,4096,4): self.file.seek(index) @@ -69,17 +77,17 @@ def parse_header(self): timestamp = unpack(">I", self.file.read(4)) x = (index/4) % 32 z = int(index/4)/32 - if offset < 2 and offset != 0: # chunk inside the header of the region file - status = -2 + if offset < 2 and offset != 0: + status = self.ChunkInHeader - elif (offset + length)*4 > self.size: # chunk outside of file - status = -1 + elif (offset + length)*4 > self.size: + status = self.ChunkOutOfFile - elif offset == 0: # no created yet - status = 1 + elif offset == 0: + status = self.ChunkNotCreated else: - status = 0 # everything ok + status = self.ChunkOK self.header[x,z] = (offset, length, timestamp, status) @@ -88,12 +96,12 @@ def parse_chunk_headers(self): for z in range(32): offset, region_header_length, timestamp, status = self.header[x,z] - if status == 1: # chunk not created yet + if status == self.ChunkNotCreated: length = None compression = None - status = None + chunk_status = self.ChunkNotCreated - elif status == 0: # there is a chunk! + elif status == self.ChunkOK: self.file.seek(offset*4096) # offset comes in sectors of 4096 bytes length = unpack(">I", self.file.read(4)) length = length[0] # unpack always returns a tuple, even unpacking one element @@ -101,30 +109,31 @@ def parse_chunk_headers(self): compression = compression[0] # TODO TODO TODO check if the region_file_length and the chunk header length are compatible if length == 0: # chunk can't be zero length - status = -3 + chunk_status = self.ChunkZeroLength else: - status = 0 + chunk_status = self.ChunkOK - elif status == -1: # error, chunk partially/completely outside the file + elif status == self.ChunkOutOfFile: if offset*4096 + 5 < self.size: # if possible read it, just in case it's useful self.file.seek(offset*4096) # offset comes in sectors of 4096 bytes length = unpack(">I", self.file.read(4)) length = length[0] # unpack always returns a tuple, even unpacking one element compression = unpack(">B",self.file.read(1)) compression = compression[0] + status = self.ChunkOutOfFile else: length = None compression = None - status = -1 + chunk_status = self.ChunkOutOfFile - elif status == -2: # error, chunk in the header of the region file + elif status == self.ChunkInHeader: length = None compression = None - status = -2 + chunk_status = self.ChunkInHeader - self.chunk_headers[x, z] = (length, compression, status) + self.chunk_headers[x, z] = (length, compression, chunk_status) def locate_free_space(self): @@ -157,15 +166,15 @@ def get_chunk(self, x, z): if region_header_status == 1: return None - elif region_header_status == -2: + elif region_header_status == self.ChunkInHeader: raise RegionHeaderError('The chunk is in the region header') - elif region_header_status == -1: + elif region_header_status == self.ChunkOutOfFile: raise RegionHeaderError('The chunk is partially/completely outside the file') - elif region_header_status == 0: + elif region_header_status == self.ChunkOK: length, compression, chunk_header_status = self.chunk_headers[x, z] - if chunk_header_status == -3: # no chunk can be 0 length! + if chunk_header_status == self.ChunkZeroLength: raise ChunkHeaderError('The length of the chunk is 0') self.file.seek(offset*4*1024 + 5) # offset comes in sectors of 4096 bytes + length bytes + compression byte @@ -174,11 +183,10 @@ def get_chunk(self, x, z): if (compression == 2): try: chunk = zlib.decompress(chunk) + chunk = StringIO(chunk) + return NBTFile(buffer=chunk) # pass uncompressed except Exception, e: raise ChunkDataError(str(e)) - - chunk = StringIO(chunk) - return NBTFile(buffer=chunk) # pass uncompressed elif (compression == 1): chunk = StringIO(chunk) @@ -188,7 +196,7 @@ def get_chunk(self, x, z): raise ChunkDataError(str(e)) else: - raise ChunkDataError('Unknown chunk compression') + raise ChunkDataError('Unknown chunk compression/format') else: return None From 0701c53f56c1d7b672d68bfe43d1a4b130bcc308 Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Sat, 16 Apr 2011 11:41:49 +0200 Subject: [PATCH 31/40] Finish the adding constants to keep track of error codes. --- nbt/region.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index 28eb8c2..cc1b7bd 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -214,7 +214,8 @@ def write_chunk(self, x, z, nbt_file): #if it will fit back in it's original slot: offset, length, timestamp, status = self.header[x, z] pad_end = False - if status in (1,-1,-2): # don't trust bad headers, write at the end. + if status in (self.ChunkNotCreated,self.ChunkOutOfFile,self.ChunkInHeader): + # don't trust bad headers, write at the end. # This chunk hasn't been generated yet, or the header is wrong # This chunk should just be appended to the end of the file self.file.seek(0,2) # go to the end of the file @@ -223,7 +224,7 @@ def write_chunk(self, x, z, nbt_file): sector = total_sectors+1 pad_end = True elif status == 0: - # TODO TODO TODO Check if chunk_status says that the lengths are incompatible (status = -3) + # TODO TODO TODO Check if chunk_status says that the lengths are incompatible (status = self.ChunkZeroLength) if nsectors <= length: sector = offset else: From eff4347c0a0187edb9c83f6b6054baac139ee941 Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Sat, 16 Apr 2011 12:18:50 +0200 Subject: [PATCH 32/40] Constants should be above parser. --- nbt/region.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index cc1b7bd..fe388d6 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -39,15 +39,9 @@ def __init__(self, filename=None, fileobj=None): self.file = open(filename, 'r+b') if fileobj: self.file = fileobj - self.chunks = [] - self.header = {} - self.chunk_headers = {} - self.extents = None - if self.file: - self.size = getsize(self.filename) - self.parse_header() - self.parse_chunk_headers() - + + # Some variables and constants + # # Status is a number representing: # -3 = Error, chunk header has a 0 length # -2 = Error, chunk inside the header of the region file @@ -60,6 +54,16 @@ def __init__(self, filename=None, fileobj=None): self.ChunkOutOfFile = -1 self.ChunkOK = 0 self.ChunkNotCreated = 1 + + self.chunks = [] + self.header = {} + self.chunk_headers = {} + self.extents = None + if self.file: + self.size = getsize(self.filename) + self.parse_header() + self.parse_chunk_headers() + def __del__(self): if self.file: From c6237a69b0cbb2c96abcd37f0eee77d36f845a81 Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Sun, 17 Apr 2011 00:38:38 +0200 Subject: [PATCH 33/40] Change constants to all caps, and add status prefix. --- nbt/region.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index fe388d6..cd1be4c 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -49,11 +49,11 @@ def __init__(self, filename=None, fileobj=None): # 0 = Ok # 1 = Chunk non-existant yet - self.ChunkZeroLength = -3 - self.ChunkInHeader = -2 - self.ChunkOutOfFile = -1 - self.ChunkOK = 0 - self.ChunkNotCreated = 1 + self.STATUS_CHUNK_ZERO_LENGTH = -3 + self.STATUS_CHUNK_IN_HEADER = -2 + self.STATUS_CHUNK_OUT_OF_FILE = -1 + self.STATUS_CHUNK_OK = 0 + self.STATUS_CHUNK_NOT_CREATED = 1 self.chunks = [] self.header = {} @@ -82,16 +82,16 @@ def parse_header(self): x = (index/4) % 32 z = int(index/4)/32 if offset < 2 and offset != 0: - status = self.ChunkInHeader + status = self.STATUS_CHUNK_IN_HEADER elif (offset + length)*4 > self.size: - status = self.ChunkOutOfFile + status = self.STATUS_CHUNK_OUT_OF_FILE elif offset == 0: - status = self.ChunkNotCreated + status = self.STATUS_CHUNK_OUT_OF_FILE else: - status = self.ChunkOK + status = self.STATUS_CHUNK_OK self.header[x,z] = (offset, length, timestamp, status) @@ -103,9 +103,9 @@ def parse_chunk_headers(self): if status == self.ChunkNotCreated: length = None compression = None - chunk_status = self.ChunkNotCreated + chunk_status = self.STATUS_CHUNK_NOT_CREATED - elif status == self.ChunkOK: + elif status == self.STATUS_CHUNK_OK: self.file.seek(offset*4096) # offset comes in sectors of 4096 bytes length = unpack(">I", self.file.read(4)) length = length[0] # unpack always returns a tuple, even unpacking one element @@ -118,7 +118,7 @@ def parse_chunk_headers(self): else: chunk_status = self.ChunkOK - elif status == self.ChunkOutOfFile: + elif status == self.STATUS_CHUNK_OUT_OF_FILE: if offset*4096 + 5 < self.size: # if possible read it, just in case it's useful self.file.seek(offset*4096) # offset comes in sectors of 4096 bytes length = unpack(">I", self.file.read(4)) @@ -132,7 +132,7 @@ def parse_chunk_headers(self): compression = None chunk_status = self.ChunkOutOfFile - elif status == self.ChunkInHeader: + elif status == self.STATUS_CHUNK_IN_HEADER: length = None compression = None chunk_status = self.ChunkInHeader @@ -170,15 +170,15 @@ def get_chunk(self, x, z): if region_header_status == 1: return None - elif region_header_status == self.ChunkInHeader: + elif region_header_status == self.STATUS_CHUNK_IN_HEADER: raise RegionHeaderError('The chunk is in the region header') - elif region_header_status == self.ChunkOutOfFile: + elif region_header_status == self.STATUS_CHUNK_OUT_OF_FILE: raise RegionHeaderError('The chunk is partially/completely outside the file') - elif region_header_status == self.ChunkOK: + elif region_header_status == self.STATUS_CHUNK_OK: length, compression, chunk_header_status = self.chunk_headers[x, z] - if chunk_header_status == self.ChunkZeroLength: + if chunk_header_status == self.STATUS_CHUNK_ZERO_LENGTH: raise ChunkHeaderError('The length of the chunk is 0') self.file.seek(offset*4*1024 + 5) # offset comes in sectors of 4096 bytes + length bytes + compression byte @@ -218,7 +218,7 @@ def write_chunk(self, x, z, nbt_file): #if it will fit back in it's original slot: offset, length, timestamp, status = self.header[x, z] pad_end = False - if status in (self.ChunkNotCreated,self.ChunkOutOfFile,self.ChunkInHeader): + if status in (self.STATUS_CHUNK_NOT_CREATED, self.STATUS_CHUNK_OUT_OF_FILE, self.STATUS_CHUNK_IN_HEADER): # don't trust bad headers, write at the end. # This chunk hasn't been generated yet, or the header is wrong # This chunk should just be appended to the end of the file @@ -227,8 +227,8 @@ def write_chunk(self, x, z, nbt_file): total_sectors = file_length/4096 sector = total_sectors+1 pad_end = True - elif status == 0: - # TODO TODO TODO Check if chunk_status says that the lengths are incompatible (status = self.ChunkZeroLength) + elif status == self.STATUS_CHUNK_OK: + # TODO TODO TODO Check if chunk_status says that the lengths are incompatible (status = self.STATUS_CHUNK_ZERO_LENGTH) if nsectors <= length: sector = offset else: From 7944421fc834442523b6082be0a5f88b4feea786 Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Sun, 17 Apr 2011 00:49:02 +0200 Subject: [PATCH 34/40] Missed a few! Renamed wrong variable name. --- nbt/region.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index cd1be4c..2e4ed39 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -100,7 +100,7 @@ def parse_chunk_headers(self): for z in range(32): offset, region_header_length, timestamp, status = self.header[x,z] - if status == self.ChunkNotCreated: + if status == self.STATUS_CHUNK_NOT_CREATED: length = None compression = None chunk_status = self.STATUS_CHUNK_NOT_CREATED @@ -113,10 +113,10 @@ def parse_chunk_headers(self): compression = compression[0] # TODO TODO TODO check if the region_file_length and the chunk header length are compatible if length == 0: # chunk can't be zero length - chunk_status = self.ChunkZeroLength + chunk_status = self.STATUS_CHUNK_ZERO_LENGTH else: - chunk_status = self.ChunkOK + chunk_status = self.STATUS_CHUNK_OK elif status == self.STATUS_CHUNK_OUT_OF_FILE: if offset*4096 + 5 < self.size: # if possible read it, just in case it's useful @@ -125,17 +125,17 @@ def parse_chunk_headers(self): length = length[0] # unpack always returns a tuple, even unpacking one element compression = unpack(">B",self.file.read(1)) compression = compression[0] - status = self.ChunkOutOfFile + chunk_status = self.STATUS_CHUNK_OUT_OF_FILE else: length = None compression = None - chunk_status = self.ChunkOutOfFile + chunk_status = self.STATUS_CHUNK_OUT_OF_FILE elif status == self.STATUS_CHUNK_IN_HEADER: length = None compression = None - chunk_status = self.ChunkInHeader + chunk_status = self.STATUS_CHUNK_IN_HEADER self.chunk_headers[x, z] = (length, compression, chunk_status) From e29199f58b264320e2677a8c89dfb01fc2f5702a Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Sun, 17 Apr 2011 10:11:50 +0200 Subject: [PATCH 35/40] Fix bad status choice. --- nbt/region.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbt/region.py b/nbt/region.py index 2e4ed39..c047152 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -88,7 +88,7 @@ def parse_header(self): status = self.STATUS_CHUNK_OUT_OF_FILE elif offset == 0: - status = self.STATUS_CHUNK_OUT_OF_FILE + status = self.STATUS_CHUNK_NOT_CREATED else: status = self.STATUS_CHUNK_OK From cb2a02cb5506fde0a2d27be9cac1305f3e378717 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Tue, 26 Apr 2011 16:46:28 -0500 Subject: [PATCH 36/40] Chunk class handling tweaks --- nbt/chunk.py | 2 +- nbt/region.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nbt/chunk.py b/nbt/chunk.py index af49c40..0e4c46d 100644 --- a/nbt/chunk.py +++ b/nbt/chunk.py @@ -112,7 +112,7 @@ def get_map(self): return im def __repr__(self): - return "("+str(self.coords[0])+","+str(self.coords[1])+")" + return "Chunk("+str(self.coords[0])+","+str(self.coords[1])+")" """ Convenience class for dealing with a Block/data byte array """ class BlockArray(object): diff --git a/nbt/region.py b/nbt/region.py index 2e4ed39..6200fd9 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -152,7 +152,7 @@ def get_chunks(self): if offset: x = (index/4) % 32 z = int(index/4)/32 - chunks.append(Chunk(x,z,length)) + chunks.append({'x':x,'z':z,'length':length}) index += 4 return chunks From 73b2ae4ad8ab794697f2c608167248ce45487bdb Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Tue, 26 Apr 2011 16:48:06 -0500 Subject: [PATCH 37/40] Parse a world to determine block composition --- block_analysis.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 block_analysis.py diff --git a/block_analysis.py b/block_analysis.py new file mode 100644 index 0000000..6873cc1 --- /dev/null +++ b/block_analysis.py @@ -0,0 +1,114 @@ +from nbt.region import RegionFile +from nbt.chunk import Chunk +import locale, os, sys + +if (len(sys.argv) == 1): + print "No world folder specified!" + sys.exit() + +world_folder = sys.argv[1] +if (not os.path.exists(world_folder)): + print "No such folder as "+filename + sys.exit() + +if (world_folder[-1] == '/'): + world_folder = world_folder[:-1] # Trim trailing slash + +regions = os.listdir(world_folder+'/region/') + +start = None +stop = None +if (len(sys.argv) == 4): + # A min/max corner was specified + start_str = sys.argv[2][1:-1] # Strip parenthesis... + start = tuple(start_str.split(',')) # and convert to tuple + stop_str = sys.argv[3][1:-1] # Strip parenthesis... + stop = tuple(stop_str.split(',')) # and convert to tuple + +block_totals = [0]*255 # up to 255 block types +try: + for filename in regions: + print "Parsing",filename,"..." + pieces = filename.split('.') + rx = int(pieces[1]) + rz = int(pieces[2]) + + # Does the region overlap the bounding box at all? + if (start != None): + if ( (rx+1)*512-1 < int(start[0]) or (rz+1)*512-1 < int(start[2]) ): + continue + elif (stop != None): + if ( rx*512-1 > int(stop[0]) or rz*512-1 > int(stop[2]) ): + continue + + file = RegionFile(filename=world_folder+'/region/'+filename) + + # Get all chunks + chunks = file.get_chunks() + for c in chunks: + # Does the chunk overlap the bounding box at all? + if (start != None): + if ( (c['x']+1)*16 + rx*512 - 1 < int(start[0]) or (c['z']+1)*16 + rz*512 - 1 < int(start[2]) ): + continue + elif (stop != None): + if ( c['x']*16 + rx*512 - 1 > int(stop[0]) or c['z']*16 + rz*512 - 1 > int(stop[2]) ): + continue + + chunk = Chunk(file.get_chunk(c['x'], c['z'])) + #print "Parsing chunk ("+str(c['x'])+", "+str(c['z'])+")" + + # Parse the blocks + for z in range(16): + world_z = z + c['z']*16 + rz*512 + if ( (start != None and world_z < int(start[2])) or (stop != None and world_z > int(stop[2])) ): + # Outside the bounding box; skip to next iteration + #print "Z break:",world_z,start[2],stop[2] + break + for x in range(16): + world_x = x + c['x']*16 + rx*512 + if ( (start != None and world_x < int(start[0])) or (stop != None and world_x > int(stop[0])) ): + # Outside the bounding box; skip to next iteration + #print "X break:",world_x,start[0],stop[0] + break + for y in range(128): + if ( (start != None and y < int(start[1])) or (stop != None and y > int(stop[1])) ): + # Outside the bounding box; skip to next iteration + #print "Y break:",y,start[1],stop[1] + break + + #print "Chunk:",c['x'], c['z'],"Coord:",x,y,z + block_id = chunk.blocks.get_block(x,y,z) + block_totals[block_id] += 1 +except KeyboardInterrupt: + print block_totals + raise + +print block_totals + +# Analyze blocks +locale.setlocale(locale.LC_ALL, 'en_US') + +total_blocks = sum(block_totals) +solid_blocks = total_blocks - block_totals[0] +solid_ratio = (solid_blocks+0.0)/total_blocks if (total_blocks > 0) else 0 +print locale.format("%d", total_blocks, grouping=True),'total blocks in region,',locale.format("%d", solid_blocks, grouping=True),"are solid ({0:0.4%})".format(solid_ratio) + +# Find valuable blocks +print 'Diamond Ore:', locale.format("%d", block_totals[56], grouping=True) +print 'Gold Ore:', locale.format("%d", block_totals[14], grouping=True) +print 'Redstone Ore:', locale.format("%d", block_totals[73], grouping=True) +print 'Iron Ore:', locale.format("%d", block_totals[15], grouping=True) +print 'Coal Ore:', locale.format("%d", block_totals[16], grouping=True) +print 'Lapis Lazuli Ore:', locale.format("%d", block_totals[21], grouping=True) +print 'Dungeons:', locale.format("%d", block_totals[52], grouping=True) + +print 'Clay:', locale.format("%d", block_totals[82], grouping=True) +print 'Sugar Cane:', locale.format("%d", block_totals[83], grouping=True) +print 'Cacti:', locale.format("%d", block_totals[81], grouping=True) +print 'Pumpkin:', locale.format("%d", block_totals[86], grouping=True) +print 'Dandelion:', locale.format("%d", block_totals[37], grouping=True) +print 'Rose:', locale.format("%d", block_totals[38], grouping=True) +print 'Brown Mushroom:', locale.format("%d", block_totals[39], grouping=True) +print 'Red Mushroom:', locale.format("%d", block_totals[40], grouping=True) +print 'Lava Springs:', locale.format("%d", block_totals[11], grouping=True) + From 1cd1e50b54a42e1fcc42ce1086e4442027d67803 Mon Sep 17 00:00:00 2001 From: Brooks Boyd Date: Fri, 6 May 2011 14:32:55 -0500 Subject: [PATCH 38/40] Fix header parsing errors First check to see if chunk exists and set appropriate flag, before checking to see if the chunk has an invalid offset (zero). Also use object constant to define not created status in get_chunk function. --- nbt/region.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index c047152..946f747 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -81,15 +81,15 @@ def parse_header(self): timestamp = unpack(">I", self.file.read(4)) x = (index/4) % 32 z = int(index/4)/32 - if offset < 2 and offset != 0: + if offset == 0 and length == 0: + status = self.STATUS_CHUNK_NOT_CREATED + + elif offset < 2 and offset != 0: status = self.STATUS_CHUNK_IN_HEADER elif (offset + length)*4 > self.size: status = self.STATUS_CHUNK_OUT_OF_FILE - elif offset == 0: - status = self.STATUS_CHUNK_NOT_CREATED - else: status = self.STATUS_CHUNK_OK @@ -167,7 +167,7 @@ def get_timestamp(self, x, z): def get_chunk(self, x, z): #read metadata block offset, length, timestamp, region_header_status = self.header[x, z] - if region_header_status == 1: + if region_header_status == self.STATUS_CHUNK_NOT_CREATED: return None elif region_header_status == self.STATUS_CHUNK_IN_HEADER: From 6e504f04a5904012e54faeade828a6610632d8de Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Mon, 20 Jun 2011 02:16:33 +0200 Subject: [PATCH 39/40] Fix bad choice of status. The sectors size was wrong. --- nbt/region.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nbt/region.py b/nbt/region.py index ee510d2..adfc002 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -87,7 +87,8 @@ def parse_header(self): elif offset < 2 and offset != 0: status = self.STATUS_CHUNK_IN_HEADER - elif (offset + length)*4 > self.size: + # (don't forget!) offset and length comes in sectors of 4096 bytes + elif (offset + length)*4096 > self.size: status = self.STATUS_CHUNK_OUT_OF_FILE else: From f2732b11b367f6721a66730e3b2b57df25f773dc Mon Sep 17 00:00:00 2001 From: Alejandro Aguilera Date: Tue, 6 Sep 2011 14:30:21 +0200 Subject: [PATCH 40/40] Add a special case for empty region files. Some region files are 0 in size, an Minecraft seems to handle them without problems. This code handle them as empty region files. --- nbt/region.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nbt/region.py b/nbt/region.py index adfc002..4e726ff 100644 --- a/nbt/region.py +++ b/nbt/region.py @@ -61,8 +61,17 @@ def __init__(self, filename=None, fileobj=None): self.extents = None if self.file: self.size = getsize(self.filename) - self.parse_header() - self.parse_chunk_headers() + if self.size == 0: + # Some region files seems to have 0 bytes of size, and + # Minecraft handle them without problems. Take them + # as empty region files. + for x in range(32): + for z in range(32): + self.header[x,z] = (0, 0, 0, self.STATUS_CHUNK_NOT_CREATED) + self.parse_chunk_headers() + else: + self.parse_header() + self.parse_chunk_headers() def __del__(self):