From dc54342b51f1e857bd9774be7df35771f1c13e5c Mon Sep 17 00:00:00 2001 From: jonathan lung Date: Tue, 1 Dec 2020 01:43:48 -0500 Subject: [PATCH 1/3] Clean up formatting. --- pyvhd.py | 109 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 60 insertions(+), 49 deletions(-) diff --git a/pyvhd.py b/pyvhd.py index 75050d5..87919e3 100755 --- a/pyvhd.py +++ b/pyvhd.py @@ -26,17 +26,19 @@ # # This creates the dynamic VHD format but does not make any attempt to detect # large zereoed sections of disk. As a result, the output is slightly larger -# than the input RAW file. +# than the input RAW file. import struct import sys import uuid import math + def divro(num, den): # Divide always rounding up and returning an integer # Is there some nicer way to do this? - return int(math.ceil((1.0*num)/(1.0*den))) + return int(math.ceil((1.0 * num) / (1.0 * den))) + def vhd_checksum(string): # This is the checksum defined in the MS spec @@ -46,6 +48,7 @@ def vhd_checksum(string): checksum += ord(byte) return (checksum ^ 0xFFFFFFFF) + def vhd_chs(size): # CHS calculation as defined by the VHD spec sectors = divro(size, SECTORSIZE) @@ -79,12 +82,14 @@ def vhd_chs(size): return (cylinders, heads, spt) + def zerostring(len): zs = "" for i in range(1, len): zs += '\0' return zs + # Header/Footer - From MS doco # 512 bytes - early versions had a 511 byte footer for no obvious reason @@ -136,7 +141,7 @@ def zerostring(len): DYNAMIC_FMT = ">8sQQIIII16sII512s192s256s" -# BAT header +# BAT header # This is not in the Microsoft spec but is present in the Xen code # This is a bitmap of the BAT where "1" indicates the BAT entry is valid # and "0" indicates that it is unallocated. @@ -149,98 +154,102 @@ def zerostring(len): BAT_HDR_FMT = ">8sQIII" -VHD_BLOCKSIZE = 2 * 1024 * 1024 # Default blocksize 2 MB +VHD_BLOCKSIZE = 2 * 1024 * 1024 # Default blocksize 2 MB SECTORSIZE = 512 -VHD_BLOCKSIZE_SECTORS = VHD_BLOCKSIZE/SECTORSIZE +VHD_BLOCKSIZE_SECTORS = VHD_BLOCKSIZE / SECTORSIZE VHD_HEADER_SIZE = struct.calcsize(HEADER_FMT) VHD_DYN_HEADER_SIZE = struct.calcsize(DYNAMIC_FMT) SECTOR_BITMAP_SIZE = VHD_BLOCKSIZE / SECTORSIZE / 8 FULL_SECTOR_BITMAP = "" -for i in range(0,SECTOR_BITMAP_SIZE): +for i in range(0, SECTOR_BITMAP_SIZE): FULL_SECTOR_BITMAP += chr(0xFF) SECTOR_BITMAP_SECTORS = divro(SECTOR_BITMAP_SIZE, SECTORSIZE) # vhd-util has a bug that pads an additional 7 sectors on to each bloc # at the end. I suspect this is due to miscalculating the size of the # bitmap. Specifically, forgetting to divide the number of bits by 8. -BLOCK_PAD_SECTORS = 7 # Bug for bug compat with vhd-util +BLOCK_PAD_SECTORS = 7 # Bug for bug compat with vhd-util + def do_vhd_convert(infile, outfile): # infile - open file object containing raw input flie # outfile - open for writing file object to which output is written - infile.seek(0,2) + infile.seek(0, 2) insize = infile.tell() infile.seek(0) bat_entries = divro(insize, VHD_BLOCKSIZE) # Block Allocation Table (BAT) size in sectors - bat_sectors = divro(bat_entries*4, SECTORSIZE) + bat_sectors = divro(bat_entries * 4, SECTORSIZE) # OK - cannot quite figure out why vhd-util adds more # pad than needed in some cases - I will just put the # first block safely past the batmap - batmap_size_sectors = divro(divro(bat_entries,8),SECTORSIZE) + batmap_size_sectors = divro(divro(bat_entries, 8), SECTORSIZE) first_block_sector = 3 + bat_sectors + 1 + batmap_size_sectors current_block_sector = first_block_sector total_block_sectors = SECTOR_BITMAP_SECTORS + VHD_BLOCKSIZE_SECTORS + BLOCK_PAD_SECTORS - bat_values = [ ] - for i in range(0,bat_entries): - bat_values.append(current_block_sector) - current_block_sector += total_block_sectors + bat_values = [] + for i in range(0, bat_entries): + bat_values.append(current_block_sector) + current_block_sector += total_block_sectors - bat="" + bat = "" for sector in bat_values: - bat += struct.pack(">I", ( sector ) ) + bat += struct.pack(">I", (sector)) # The Xen code adds yet another sector map, this one of the BAT itself. # This converter code pre-allocates everything, so we just need a string # full of set bits of the correct size - batmap="" - for i in range(0,bat_entries/8): - batmap += chr(0xFF) + batmap = "" + for i in range(0, bat_entries / 8): + batmap += chr(0xFF) extra_bits = bat_entries % 8 if extra_bits != 0: - batmap += chr((0xFF << (8-extra_bits)) & 0xFF) + batmap += chr((0xFF << (8 - extra_bits)) & 0xFF) cookie3 = "tdbatmap" # 3 sectors for the other headers plus one sector for this batmap_offset = (3 + bat_sectors + 1) * SECTORSIZE batmap_size = batmap_size_sectors - batmap_version = 0x00010002 # from vhd-util + batmap_version = 0x00010002 # from vhd-util batmap_checksum = vhd_checksum(batmap) - batmap_vals = ( cookie3, batmap_offset, batmap_size, batmap_version, batmap_checksum) + batmap_vals = (cookie3, batmap_offset, batmap_size, batmap_version, + batmap_checksum) batmap_hdr = struct.pack(BAT_HDR_FMT, *batmap_vals) batmap_hdr_location = 3 + bat_sectors cookie = "conectix" - features = 2 # Set by convention - means nothing + features = 2 # Set by convention - means nothing fmt_version = 0x00010000 - data_offset = 512 # location of dynamic header + data_offset = 512 # location of dynamic header # This is a problematic field - vhd-util interprets it as local # time and will reject images that have a stamp in the future # set it to 24 hours ago to be safe or EPOCH (zero) to be safer - timestamp = 0 + timestamp = 0 creator_app = "tap" - creator_ver = 0x10003 # match vhd-util - creator_os = 0 # match vhd-util + creator_ver = 0x10003 # match vhd-util + creator_os = 0 # match vhd-util orig_size = insize curr_size = insize (disk_c, disk_h, disk_s) = vhd_chs(curr_size) - disk_type = 3 # Dynamic - checksum = 0 # calculated later + disk_type = 3 # Dynamic + checksum = 0 # calculated later my_uuid = uuid.uuid4().get_bytes() - saved_state= 0 + saved_state = 0 reserved = zerostring(427) - header_vals = [ cookie, features, fmt_version, data_offset, timestamp, - creator_app, creator_ver, creator_os, orig_size, curr_size, disk_c, disk_h, - disk_s, disk_type, checksum, my_uuid, saved_state, reserved ] + header_vals = [ + cookie, features, fmt_version, data_offset, timestamp, creator_app, + creator_ver, creator_os, orig_size, curr_size, disk_c, disk_h, disk_s, + disk_type, checksum, my_uuid, saved_state, reserved + ] header = struct.pack(HEADER_FMT, *tuple(header_vals)) @@ -251,29 +260,30 @@ def do_vhd_convert(infile, outfile): final_header = struct.pack(HEADER_FMT, *tuple(header_vals)) cookie2 = "cxsparse" - data_offset2 = 0xFFFFFFFFFFFFFFFF + data_offset2 = 0xFFFFFFFFFFFFFFFF table_offset = 1536 - header_version = 0x00010000 # match vhd-util + header_version = 0x00010000 # match vhd-util max_table_entries = bat_entries block_size = VHD_BLOCKSIZE - checksum2 = 0 # calculated later - parent_uuid=zerostring(16) + checksum2 = 0 # calculated later + parent_uuid = zerostring(16) parent_timestamp = 0 reserved2 = 0 parent_name = zerostring(512) parent_locents = zerostring(192) reserved3 = zerostring(256) - dyn_vals = [ cookie2, data_offset2, table_offset, header_version, - max_table_entries, block_size, checksum2, parent_uuid, parent_timestamp, - reserved2, parent_name, parent_locents, reserved3 ] + dyn_vals = [ + cookie2, data_offset2, table_offset, header_version, max_table_entries, + block_size, checksum2, parent_uuid, parent_timestamp, reserved2, + parent_name, parent_locents, reserved3 + ] dyn_header = struct.pack(DYNAMIC_FMT, *tuple(dyn_vals)) checksum2 = vhd_checksum(dyn_header) dyn_vals[6] = checksum2 final_dyn_header = struct.pack(DYNAMIC_FMT, *tuple(dyn_vals)) - outfile.write(final_header) outfile.write(final_dyn_header) outfile.write(bat) @@ -283,14 +293,17 @@ def do_vhd_convert(infile, outfile): outfile.write(batmap) for block_start in bat_values: - outfile.seek(block_start * SECTORSIZE) - outfile.write(FULL_SECTOR_BITMAP) - outfile.seek( (SECTOR_BITMAP_SECTORS + block_start) * SECTORSIZE) - outfile.write(infile.read(VHD_BLOCKSIZE)) - - outfile.seek( (block_start + SECTOR_BITMAP_SECTORS + VHD_BLOCKSIZE_SECTORS) * SECTORSIZE) + outfile.seek(block_start * SECTORSIZE) + outfile.write(FULL_SECTOR_BITMAP) + outfile.seek((SECTOR_BITMAP_SECTORS + block_start) * SECTORSIZE) + outfile.write(infile.read(VHD_BLOCKSIZE)) + + outfile.seek( + (block_start + SECTOR_BITMAP_SECTORS + VHD_BLOCKSIZE_SECTORS) * + SECTORSIZE) outfile.write(final_header) + if __name__ == '__main__': if len(sys.argv) != 3: print "usage: %s " % sys.argv[0] @@ -300,5 +313,3 @@ def do_vhd_convert(infile, outfile): do_vhd_convert(infile, outfile) infile.close() outfile.close() - - From 907d4b7dc019249d587febc08b41381be5fede58 Mon Sep 17 00:00:00 2001 From: jonathan lung Date: Tue, 1 Dec 2020 01:46:06 -0500 Subject: [PATCH 2/3] Python 2 to Python 3. --- pyvhd.py | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/pyvhd.py b/pyvhd.py index 87919e3..d91c169 100755 --- a/pyvhd.py +++ b/pyvhd.py @@ -37,7 +37,7 @@ def divro(num, den): # Divide always rounding up and returning an integer # Is there some nicer way to do this? - return int(math.ceil((1.0 * num) / (1.0 * den))) + return int(math.ceil(num / den)) def vhd_checksum(string): @@ -45,7 +45,7 @@ def vhd_checksum(string): # sum up all bytes in the checked structure then take the ones compliment checksum = 0 for byte in string: - checksum += ord(byte) + checksum += byte return (checksum ^ 0xFFFFFFFF) @@ -58,35 +58,35 @@ def vhd_chs(size): if sectors >= 65535 * 16 * 63: spt = 255 - cth = sectors / spt + cth = sectors // spt heads = 16 else: spt = 17 - cth = sectors / spt - heads = (cth + 1023) / 1024 + cth = sectors // spt + heads = (cth + 1023) // 1024 if heads < 4: heads = 4 if (cth >= (heads * 1024)) or (heads > 16): spt = 31 - cth = sectors / spt + cth = sectors // spt heads = 16 if cth >= (heads * 1024): spt = 63 - cth = sectors / spt + cth = sectors // spt heads = 16 - cylinders = cth / heads + cylinders = cth // heads return (cylinders, heads, spt) def zerostring(len): - zs = "" + zs = b'' for i in range(1, len): - zs += '\0' + zs += b'\0' return zs @@ -156,13 +156,13 @@ def zerostring(len): VHD_BLOCKSIZE = 2 * 1024 * 1024 # Default blocksize 2 MB SECTORSIZE = 512 -VHD_BLOCKSIZE_SECTORS = VHD_BLOCKSIZE / SECTORSIZE +VHD_BLOCKSIZE_SECTORS = VHD_BLOCKSIZE // SECTORSIZE VHD_HEADER_SIZE = struct.calcsize(HEADER_FMT) VHD_DYN_HEADER_SIZE = struct.calcsize(DYNAMIC_FMT) -SECTOR_BITMAP_SIZE = VHD_BLOCKSIZE / SECTORSIZE / 8 -FULL_SECTOR_BITMAP = "" +SECTOR_BITMAP_SIZE = VHD_BLOCKSIZE // SECTORSIZE // 8 +FULL_SECTOR_BITMAP = b'' for i in range(0, SECTOR_BITMAP_SIZE): - FULL_SECTOR_BITMAP += chr(0xFF) + FULL_SECTOR_BITMAP += b'\xff' SECTOR_BITMAP_SECTORS = divro(SECTOR_BITMAP_SIZE, SECTORSIZE) # vhd-util has a bug that pads an additional 7 sectors on to each bloc # at the end. I suspect this is due to miscalculating the size of the @@ -195,7 +195,7 @@ def do_vhd_convert(infile, outfile): bat_values.append(current_block_sector) current_block_sector += total_block_sectors - bat = "" + bat = b'' for sector in bat_values: bat += struct.pack(">I", (sector)) @@ -203,15 +203,15 @@ def do_vhd_convert(infile, outfile): # This converter code pre-allocates everything, so we just need a string # full of set bits of the correct size - batmap = "" - for i in range(0, bat_entries / 8): + batmap = b'' + for i in range(0, bat_entries // 8): batmap += chr(0xFF) extra_bits = bat_entries % 8 if extra_bits != 0: - batmap += chr((0xFF << (8 - extra_bits)) & 0xFF) + batmap += bytes([(0xFF << (8 - extra_bits)) & 0xFF]) - cookie3 = "tdbatmap" + cookie3 = b"tdbatmap" # 3 sectors for the other headers plus one sector for this batmap_offset = (3 + bat_sectors + 1) * SECTORSIZE batmap_size = batmap_size_sectors @@ -225,7 +225,7 @@ def do_vhd_convert(infile, outfile): batmap_hdr_location = 3 + bat_sectors - cookie = "conectix" + cookie = b"conectix" features = 2 # Set by convention - means nothing fmt_version = 0x00010000 data_offset = 512 # location of dynamic header @@ -233,7 +233,7 @@ def do_vhd_convert(infile, outfile): # time and will reject images that have a stamp in the future # set it to 24 hours ago to be safe or EPOCH (zero) to be safer timestamp = 0 - creator_app = "tap" + creator_app = b"tap" creator_ver = 0x10003 # match vhd-util creator_os = 0 # match vhd-util orig_size = insize @@ -241,7 +241,7 @@ def do_vhd_convert(infile, outfile): (disk_c, disk_h, disk_s) = vhd_chs(curr_size) disk_type = 3 # Dynamic checksum = 0 # calculated later - my_uuid = uuid.uuid4().get_bytes() + my_uuid = uuid.uuid4().bytes saved_state = 0 reserved = zerostring(427) @@ -259,7 +259,7 @@ def do_vhd_convert(infile, outfile): final_header = struct.pack(HEADER_FMT, *tuple(header_vals)) - cookie2 = "cxsparse" + cookie2 = b"cxsparse" data_offset2 = 0xFFFFFFFFFFFFFFFF table_offset = 1536 header_version = 0x00010000 # match vhd-util @@ -306,10 +306,10 @@ def do_vhd_convert(infile, outfile): if __name__ == '__main__': if len(sys.argv) != 3: - print "usage: %s " % sys.argv[0] + print("usage: %s " % sys.argv[0]) sys.exit(1) - infile = open(sys.argv[1], "r") - outfile = open(sys.argv[2], "w") + infile = open(sys.argv[1], "rb") + outfile = open(sys.argv[2], "wb") do_vhd_convert(infile, outfile) infile.close() outfile.close() From 5eaf93a4c9205cdc02637fce9fff055664c8a1d3 Mon Sep 17 00:00:00 2001 From: jonathan lung Date: Tue, 1 Dec 2020 01:42:41 -0500 Subject: [PATCH 3/3] Updating for PyPi. --- .gitignore | 4 ++++ COPYING => LICENSE | 0 pyvhd.py => pyvhd/__init__.py | 0 setup.py | 22 ++++++++++++++++++++++ tests/.keep | 0 5 files changed, 26 insertions(+) create mode 100644 .gitignore rename COPYING => LICENSE (100%) rename pyvhd.py => pyvhd/__init__.py (100%) create mode 100644 setup.py create mode 100644 tests/.keep diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..77e4dbf --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +build +dist +*.egg-info +*.pyc diff --git a/COPYING b/LICENSE similarity index 100% rename from COPYING rename to LICENSE diff --git a/pyvhd.py b/pyvhd/__init__.py similarity index 100% rename from pyvhd.py rename to pyvhd/__init__.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f5c09d7 --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="pyvhd", + version="0.0.1", + author="Ian McLeod", + author_email="imcleod@redhat.com", + description="Pure python code to create dynamic VHD files for Xen/Xenserver imports", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/lungj/pyvhd", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)", + "Operating System :: OS Independent", + ], + python_requires='>=3.2', +) \ No newline at end of file diff --git a/tests/.keep b/tests/.keep new file mode 100644 index 0000000..e69de29