Merge pull request #158 from nicoboss/bugfix/pfs0-header-padding

Fixed PFS0 header padding so it follows the PFS0 specification

nsz.py

@@ -10,7 +10,7 @@
 	import multiprocessing
 	multiprocessing.freeze_support()
 	try:
-		import nsp
+		import nsz
 	except ImportError:
 		path = pathlib.Path(__file__).resolve().parent.absolute()
 		sys.path.append(str(path))

nsz/BlockCompressor.py

@@ -210,7 +210,7 @@ def blockCompressNsp(filePath, compressionLevel, keepDelta, removePadding, useLo
 	Print.info(f'Block compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {nszPath}')
 
 	try:
-		with Pfs0.Pfs0Stream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp:
+		with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp:
 			blockCompressContainer(container, nsp, compressionLevel, keepDelta, useLongDistanceMode, blockSizeExponent, threads)
 	except BaseException as ex:
 		if not ex is KeyboardInterrupt:

nsz/Fs/Pfs0.py

@@ -62,32 +62,34 @@ def close(self):
 			self.seek(0)
 			self.write(self.getHeader())
 			super(Pfs0Stream, self).close()
-
-	def getHeaderSize(self):
-		stringTable = '\x00'.join(file['name'] for file in self.files)+'\x00'
-		headerSize = 0x10 + len(self.files) * 0x18 + self.stringTableSize
-		return headerSize
+
+	#0xff => 0x1, 0x100 => 0x20, 0x1ff => 0x1, 0x120 => 0x20
+	def allign0x20(self, n):
+		return 0x20-n%0x20
 
 	def getStringTableSize(self):
-		stringTable = '\x00'.join(file['name'] for file in self.files)+'\x00'
-		stringTableLen = len(stringTable)
+		stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00'
+		headerSizeNonPadded = 0x10 + len(self.files) * 0x18 + len(stringTableNonPadded)
+		stringTableSizePadded = len(stringTableNonPadded) + self.allign0x20(headerSizeNonPadded)
 		if self._stringTableSize == None:
-			self._stringTableSize = stringTableLen
-		if stringTableLen > self._stringTableSize:
-			self._stringTableSize = stringTableLen
+			self._stringTableSize = stringTableSizePadded
+		elif len(stringTableNonPadded) > self._stringTableSize:
+			self._stringTableSize = len(stringTableNonPadded)
 		return self._stringTableSize
 
 	def getFirstFileOffset(self):
 		return self.files[0].offset
 
 	def getHeader(self):
-		stringTable = '\x00'.join(file['name'] for file in self.files)
-		headerSize = 0x10 + len(self.files) * 0x18 + self.getStringTableSize()
+		stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00'
+		stringTableSizePadded = self.getStringTableSize()
+		stringTable = stringTableNonPadded + ('\x00'*(stringTableSizePadded-len(stringTableNonPadded)))
+		headerSize = 0x10 + len(self.files) * 0x18 + stringTableSizePadded
 
 		h = b''
 		h += b'PFS0'
 		h += len(self.files).to_bytes(4, byteorder='little')
-		h += (self.getStringTableSize()).to_bytes(4, byteorder='little')
+		h += (stringTableSizePadded).to_bytes(4, byteorder='little')
 		h += b'\x00\x00\x00\x00'
 
 		stringOffset = 0
@@ -140,27 +142,34 @@ def resize(self, name, size):
 	def close(self):
 		pass
 
+	#0xff => 0x1, 0x100 => 0x20, 0x1ff => 0x1, 0x120 => 0x20
+	def allign0x20(self, n):
+		return 0x20-n%0x20
+
 	def getStringTableSize(self):
-		stringTable = '\x00'.join(file['name'] for file in self.files)
-		stringTableLen = len(stringTable)
+		stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00'
+		headerSizeNonPadded = 0x10 + len(self.files) * 0x18 + len(stringTableNonPadded)
+		stringTableSizePadded = len(stringTableNonPadded) + self.allign0x20(headerSizeNonPadded)
 		if self._stringTableSize == None:
-			self._stringTableSize = stringTableLen
-		if stringTableLen > self._stringTableSize:
-			self._stringTableSize = stringTableLen
+			self._stringTableSize = stringTableSizePadded
+		elif len(stringTableNonPadded) > self._stringTableSize:
+			self._stringTableSize = len(stringTableNonPadded)
 		return self._stringTableSize
 
 	def getHash(self):
 		hexHash = self.binhash.hexdigest()
 		return hexHash
 
 	def getHeaderHash(self):
-		stringTable = '\x00'.join(file['name'] for file in self.files)
-		headerSize = 0x10 + len(self.files) * 0x18 + self.getStringTableSize()
+		stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00'
+		stringTableSizePadded = self.getStringTableSize()
+		stringTable = stringTableNonPadded + ('\x00'*(stringTableSizePadded-len(stringTableNonPadded)))
+		headerSize = 0x10 + len(self.files) * 0x18 + stringTableSizePadded
 
 		h = b''
 		h += b'PFS0'
 		h += len(self.files).to_bytes(4, byteorder='little')
-		h += (self.getStringTableSize()).to_bytes(4, byteorder='little')
+		h += (stringTableSizePadded).to_bytes(4, byteorder='little')
 		h += b'\x00\x00\x00\x00'
 
 		stringOffset = 0
@@ -191,6 +200,13 @@ def __init__(self, buffer, path = None, mode = None, cryptoType = -1, cryptoKey
 			#self.offset += sectionStart
 			#self.size -= sectionStart
 
+	#0xff => 0x1, 0x100 => 0x20, 0x1ff => 0x1, 0x120 => 0x20
+	def allign0x20(self, n):
+		return 0x20-n%0x20
+
+	def getPaddedHeaderSize(self):
+		return self._headerSize + self.allign0x20(self._headerSize);
+
 	def getHeaderSize(self):
 		return self._headerSize;
 

nsz/NszDecompressor.py

@@ -197,17 +197,17 @@ def __decompressNsz(filePath, outputDir, removePadding, write, raiseVerification
 			filePathNsp = changeExtension(filePath, '.nsp')
 			outPath = filePathNsp if outputDir == None else str(Path(outputDir).joinpath(Path(filePathNsp).name))
 			Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint)
-			with Pfs0.Pfs0Stream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), outPath) as nsp:
+			with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), outPath) as nsp:
 				__decompressContainer(container, nsp, fileHashes, True, raiseVerificationException, raisePfs0Exception, statusReportInfo, pleaseNoPrint)
 		else:
-			with Pfs0.Pfs0VerifyStream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as nsp:
+			with Pfs0.Pfs0VerifyStream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as nsp:
 				__decompressContainer(container, nsp, fileHashes, True, raiseVerificationException, raisePfs0Exception, statusReportInfo, pleaseNoPrint)
 				Print.info("[PFS0 HEAD]  " + nsp.getHeaderHash())
 				Print.info("[PFS0 DATA]  " + nsp.getHash())
 				if originalFilePath != None: 
 					originalContainer = factory(originalFilePath)
 					originalContainer.open(str(originalFilePath), 'rb')
-					with Pfs0.Pfs0VerifyStream(originalContainer.getHeaderSize() if removePadding else originalContainer.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as originalNsp:
+					with Pfs0.Pfs0VerifyStream(originalContainer.getPaddedHeaderSize() if removePadding else originalContainer.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as originalNsp:
 						__decompressContainer(originalContainer, originalNsp, fileHashes, True, raiseVerificationException, raisePfs0Exception, statusReportInfo, pleaseNoPrint)
 						Print.info("[PFS0 HEAD]  " + originalNsp.getHeaderHash())
 						Print.info("[PFS0 DATA]  " + originalNsp.getHash())

nsz/SolidCompressor.py

@@ -129,7 +129,7 @@ def solidCompressNsp(filePath, compressionLevel, keepDelta, removePadding, useLo
 	Print.info(f'Solid compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {nszPath}', pleaseNoPrint)
 
 	try:
-		with Pfs0.Pfs0Stream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp:
+		with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp:
 			processContainer(container, nsp, compressionLevel, keepDelta, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint)
 	except BaseException as ex:
 		if not ex is KeyboardInterrupt:

nsz/__init__.py

@@ -154,6 +154,9 @@ def main():
 			if args.verify and not args.quick_verify and not args.keep_delta:
 				Print.info("Warning: --verify requires --keep-delta when used during compression or it will detect removed NDV0 fragments as errors. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.3.0.")
 				args.quick_verify = True
+			if args.verify and not args.quick_verify and args.remove_padding:
+				Print.info("Warning: --verify and --remove-padding are incompatible with each others. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.6.0.")
+				args.quick_verify = True
 			sourceFileToDelete = []
 			for f_str in args.file:
 				for filePath in expandFiles(Path(f_str)):