From 2d2ff130bbec1335e51948edacd4a33995f2b149 Mon Sep 17 00:00:00 2001 From: nicoboss Date: Sat, 2 Dec 2023 17:04:59 +0100 Subject: [PATCH 1/7] Fixed PFS0 header padding so it follows the PFS0 specification. This fixes #150 --- nsz/Fs/Pfs0.py | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/nsz/Fs/Pfs0.py b/nsz/Fs/Pfs0.py index e844d46..6a45506 100644 --- a/nsz/Fs/Pfs0.py +++ b/nsz/Fs/Pfs0.py @@ -62,32 +62,34 @@ def close(self): self.seek(0) self.write(self.getHeader()) super(Pfs0Stream, self).close() - - def getHeaderSize(self): - stringTable = '\x00'.join(file['name'] for file in self.files)+'\x00' - headerSize = 0x10 + len(self.files) * 0x18 + self.stringTableSize - return headerSize + + #0xff => 0x1, 0x100 => 0x20, 0x1ff => 0x1, 0x120 => 0x20 + def allign0x20(self, n): + return 0x20-n%0x20 def getStringTableSize(self): - stringTable = '\x00'.join(file['name'] for file in self.files)+'\x00' - stringTableLen = len(stringTable) + stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00' + headerSizeNonPadded = 0x10 + len(self.files) * 0x18 + len(stringTableNonPadded) + stringTableSizePadded = len(stringTableNonPadded) + self.allign0x20(headerSizeNonPadded) if self._stringTableSize == None: - self._stringTableSize = stringTableLen - if stringTableLen > self._stringTableSize: - self._stringTableSize = stringTableLen + self._stringTableSize = stringTableSizePadded + if stringTableSizePadded > self._stringTableSize: + self._stringTableSize = stringTableSizePadded return self._stringTableSize def getFirstFileOffset(self): return self.files[0].offset def getHeader(self): - stringTable = '\x00'.join(file['name'] for file in self.files) - headerSize = 0x10 + len(self.files) * 0x18 + self.getStringTableSize() + stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00' + stringTableSizePadded = self.getStringTableSize() + stringTable = stringTableNonPadded + ('\x00'*(stringTableSizePadded-len(stringTableNonPadded))) + headerSize = 0x10 + len(self.files) * 0x18 + stringTableSizePadded h = b'' h += b'PFS0' h += len(self.files).to_bytes(4, byteorder='little') - h += (self.getStringTableSize()).to_bytes(4, byteorder='little') + h += (stringTableSizePadded).to_bytes(4, byteorder='little') h += b'\x00\x00\x00\x00' stringOffset = 0 @@ -141,12 +143,13 @@ def close(self): pass def getStringTableSize(self): - stringTable = '\x00'.join(file['name'] for file in self.files) - stringTableLen = len(stringTable) + stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00' + headerSizeNonPadded = 0x10 + len(self.files) * 0x18 + len(stringTableNonPadded) + stringTableSizePadded = len(stringTableNonPadded) + self.allign0x20(headerSizeNonPadded) if self._stringTableSize == None: - self._stringTableSize = stringTableLen - if stringTableLen > self._stringTableSize: - self._stringTableSize = stringTableLen + self._stringTableSize = stringTableSizePadded + if stringTableSizePadded > self._stringTableSize: + self._stringTableSize = stringTableSizePadded return self._stringTableSize def getHash(self): @@ -154,8 +157,10 @@ def getHash(self): return hexHash def getHeaderHash(self): - stringTable = '\x00'.join(file['name'] for file in self.files) - headerSize = 0x10 + len(self.files) * 0x18 + self.getStringTableSize() + stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00' + stringTableSizePadded = self.getStringTableSize() + stringTable = stringTableNonPadded + ('\x00'*(stringTableSizePadded-len(stringTableNonPadded))) + headerSize = 0x10 + len(self.files) * 0x18 + stringTableSizePadded h = b'' h += b'PFS0' From fe4520ce276f4cb32ea6fde3cd7c9ea213515d43 Mon Sep 17 00:00:00 2001 From: nicoboss Date: Sat, 2 Dec 2023 17:19:30 +0100 Subject: [PATCH 2/7] Added the allign0x20 function to the Pfs0VerifyStream class as well --- nsz/Fs/Pfs0.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nsz/Fs/Pfs0.py b/nsz/Fs/Pfs0.py index 6a45506..4c6f421 100644 --- a/nsz/Fs/Pfs0.py +++ b/nsz/Fs/Pfs0.py @@ -142,6 +142,10 @@ def resize(self, name, size): def close(self): pass + #0xff => 0x1, 0x100 => 0x20, 0x1ff => 0x1, 0x120 => 0x20 + def allign0x20(self, n): + return 0x20-n%0x20 + def getStringTableSize(self): stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00' headerSizeNonPadded = 0x10 + len(self.files) * 0x18 + len(stringTableNonPadded) From eb51c4528264fc4002aa9f40f7fac4c75bd4c7c1 Mon Sep 17 00:00:00 2001 From: nicoboss Date: Sat, 2 Dec 2023 20:01:55 +0100 Subject: [PATCH 3/7] We obviously want to try importing nsz and not nsp. --- nsz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nsz.py b/nsz.py index df280c8..5cfca56 100755 --- a/nsz.py +++ b/nsz.py @@ -10,7 +10,7 @@ import multiprocessing multiprocessing.freeze_support() try: - import nsp + import nsz except ImportError: path = pathlib.Path(__file__).resolve().parent.absolute() sys.path.append(str(path)) From c248d7e95e5cb34b3f1c0614b4c4c1c6aaa4bce8 Mon Sep 17 00:00:00 2001 From: nicoboss Date: Sat, 2 Dec 2023 20:04:39 +0100 Subject: [PATCH 4/7] Fixed some logic responsible to handle too short PFS0 header sizes --- nsz/Fs/Pfs0.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nsz/Fs/Pfs0.py b/nsz/Fs/Pfs0.py index 4c6f421..7bdb70d 100644 --- a/nsz/Fs/Pfs0.py +++ b/nsz/Fs/Pfs0.py @@ -73,8 +73,8 @@ def getStringTableSize(self): stringTableSizePadded = len(stringTableNonPadded) + self.allign0x20(headerSizeNonPadded) if self._stringTableSize == None: self._stringTableSize = stringTableSizePadded - if stringTableSizePadded > self._stringTableSize: - self._stringTableSize = stringTableSizePadded + elif len(stringTableNonPadded) > self._stringTableSize: + self._stringTableSize = len(stringTableNonPadded) return self._stringTableSize def getFirstFileOffset(self): @@ -152,8 +152,8 @@ def getStringTableSize(self): stringTableSizePadded = len(stringTableNonPadded) + self.allign0x20(headerSizeNonPadded) if self._stringTableSize == None: self._stringTableSize = stringTableSizePadded - if stringTableSizePadded > self._stringTableSize: - self._stringTableSize = stringTableSizePadded + elif len(stringTableNonPadded) > self._stringTableSize: + self._stringTableSize = len(stringTableNonPadded) return self._stringTableSize def getHash(self): @@ -169,7 +169,7 @@ def getHeaderHash(self): h = b'' h += b'PFS0' h += len(self.files).to_bytes(4, byteorder='little') - h += (self.getStringTableSize()).to_bytes(4, byteorder='little') + h += (stringTableSizePadded).to_bytes(4, byteorder='little') h += b'\x00\x00\x00\x00' stringOffset = 0 From 87884d1b3276f5dd9800b421b2c135a72eb14c6a Mon Sep 17 00:00:00 2001 From: nicoboss Date: Sun, 3 Dec 2023 12:11:30 +0100 Subject: [PATCH 5/7] Fixed issue with PFS0 header overlapping the first file when the --remove-padding option is used --- nsz/BlockCompressor.py | 2 +- nsz/Fs/Pfs0.py | 7 +++++++ nsz/NszDecompressor.py | 6 +++--- nsz/SolidCompressor.py | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/nsz/BlockCompressor.py b/nsz/BlockCompressor.py index 90a4650..4ef64b1 100644 --- a/nsz/BlockCompressor.py +++ b/nsz/BlockCompressor.py @@ -210,7 +210,7 @@ def blockCompressNsp(filePath, compressionLevel, keepDelta, removePadding, useLo Print.info(f'Block compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {nszPath}') try: - with Pfs0.Pfs0Stream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp: + with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp: blockCompressContainer(container, nsp, compressionLevel, keepDelta, useLongDistanceMode, blockSizeExponent, threads) except BaseException as ex: if not ex is KeyboardInterrupt: diff --git a/nsz/Fs/Pfs0.py b/nsz/Fs/Pfs0.py index 7bdb70d..151f464 100644 --- a/nsz/Fs/Pfs0.py +++ b/nsz/Fs/Pfs0.py @@ -200,6 +200,13 @@ def __init__(self, buffer, path = None, mode = None, cryptoType = -1, cryptoKey #self.offset += sectionStart #self.size -= sectionStart + #0xff => 0x1, 0x100 => 0x20, 0x1ff => 0x1, 0x120 => 0x20 + def allign0x20(self, n): + return 0x20-n%0x20 + + def getPaddedHeaderSize(self): + return self._headerSize + self.allign0x20(self._headerSize); + def getHeaderSize(self): return self._headerSize; diff --git a/nsz/NszDecompressor.py b/nsz/NszDecompressor.py index 8d1d1db..39cd2b8 100644 --- a/nsz/NszDecompressor.py +++ b/nsz/NszDecompressor.py @@ -197,17 +197,17 @@ def __decompressNsz(filePath, outputDir, removePadding, write, raiseVerification filePathNsp = changeExtension(filePath, '.nsp') outPath = filePathNsp if outputDir == None else str(Path(outputDir).joinpath(Path(filePathNsp).name)) Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint) - with Pfs0.Pfs0Stream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), outPath) as nsp: + with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), outPath) as nsp: __decompressContainer(container, nsp, fileHashes, True, raiseVerificationException, raisePfs0Exception, statusReportInfo, pleaseNoPrint) else: - with Pfs0.Pfs0VerifyStream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as nsp: + with Pfs0.Pfs0VerifyStream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as nsp: __decompressContainer(container, nsp, fileHashes, True, raiseVerificationException, raisePfs0Exception, statusReportInfo, pleaseNoPrint) Print.info("[PFS0 HEAD] " + nsp.getHeaderHash()) Print.info("[PFS0 DATA] " + nsp.getHash()) if originalFilePath != None: originalContainer = factory(originalFilePath) originalContainer.open(str(originalFilePath), 'rb') - with Pfs0.Pfs0VerifyStream(originalContainer.getHeaderSize() if removePadding else originalContainer.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as originalNsp: + with Pfs0.Pfs0VerifyStream(originalContainer.getPaddedHeaderSize() if removePadding else originalContainer.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as originalNsp: __decompressContainer(originalContainer, originalNsp, fileHashes, True, raiseVerificationException, raisePfs0Exception, statusReportInfo, pleaseNoPrint) Print.info("[PFS0 HEAD] " + originalNsp.getHeaderHash()) Print.info("[PFS0 DATA] " + originalNsp.getHash()) diff --git a/nsz/SolidCompressor.py b/nsz/SolidCompressor.py index 9d0df92..49e26a8 100644 --- a/nsz/SolidCompressor.py +++ b/nsz/SolidCompressor.py @@ -129,7 +129,7 @@ def solidCompressNsp(filePath, compressionLevel, keepDelta, removePadding, useLo Print.info(f'Solid compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {nszPath}', pleaseNoPrint) try: - with Pfs0.Pfs0Stream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp: + with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp: processContainer(container, nsp, compressionLevel, keepDelta, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint) except BaseException as ex: if not ex is KeyboardInterrupt: From 3a11ff06bc416c376d9e7680bbcba0792937b2d5 Mon Sep 17 00:00:00 2001 From: nicoboss Date: Sun, 3 Dec 2023 12:15:00 +0100 Subject: [PATCH 6/7] Warning: --verify and --remove-padding are incompatible with each other's. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.6.0. --- nsz/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nsz/__init__.py b/nsz/__init__.py index f6d727d..53d4e9d 100644 --- a/nsz/__init__.py +++ b/nsz/__init__.py @@ -154,6 +154,9 @@ def main(): if args.verify and not args.quick_verify and not args.keep_delta: Print.info("Warning: --verify requires --keep-delta when used during compression or it will detect removed NDV0 fragments as errors. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.3.0.") args.quick_verify = True + if args.verify and not args.quick_verify and not args.keep_delta: + Print.info("Warning: --verify and --remove-padding are incompatible with each others. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.6.0.") + args.quick_verify = True sourceFileToDelete = [] for f_str in args.file: for filePath in expandFiles(Path(f_str)): From 5799480fdd3f3fdc89b82c35fd0412824046444d Mon Sep 17 00:00:00 2001 From: nicoboss Date: Sun, 3 Dec 2023 12:21:43 +0100 Subject: [PATCH 7/7] Fixed --verify and --remove-padding incompatibility check --- nsz/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nsz/__init__.py b/nsz/__init__.py index 53d4e9d..ce199e1 100644 --- a/nsz/__init__.py +++ b/nsz/__init__.py @@ -154,7 +154,7 @@ def main(): if args.verify and not args.quick_verify and not args.keep_delta: Print.info("Warning: --verify requires --keep-delta when used during compression or it will detect removed NDV0 fragments as errors. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.3.0.") args.quick_verify = True - if args.verify and not args.quick_verify and not args.keep_delta: + if args.verify and not args.quick_verify and args.remove_padding: Print.info("Warning: --verify and --remove-padding are incompatible with each others. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.6.0.") args.quick_verify = True sourceFileToDelete = []