Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed PFS0 header padding so it follows the PFS0 specification #158

Merged
merged 7 commits into from
Dec 3, 2023
2 changes: 1 addition & 1 deletion nsz.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import multiprocessing
multiprocessing.freeze_support()
try:
import nsp
import nsz
except ImportError:
path = pathlib.Path(__file__).resolve().parent.absolute()
sys.path.append(str(path))
Expand Down
2 changes: 1 addition & 1 deletion nsz/BlockCompressor.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ def blockCompressNsp(filePath, compressionLevel, keepDelta, removePadding, useLo
Print.info(f'Block compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {nszPath}')

try:
with Pfs0.Pfs0Stream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp:
with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp:
blockCompressContainer(container, nsp, compressionLevel, keepDelta, useLongDistanceMode, blockSizeExponent, threads)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Expand Down
58 changes: 37 additions & 21 deletions nsz/Fs/Pfs0.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,32 +62,34 @@ def close(self):
self.seek(0)
self.write(self.getHeader())
super(Pfs0Stream, self).close()

def getHeaderSize(self):
stringTable = '\x00'.join(file['name'] for file in self.files)+'\x00'
headerSize = 0x10 + len(self.files) * 0x18 + self.stringTableSize
return headerSize

#0xff => 0x1, 0x100 => 0x20, 0x1ff => 0x1, 0x120 => 0x20
def allign0x20(self, n):
return 0x20-n%0x20

def getStringTableSize(self):
stringTable = '\x00'.join(file['name'] for file in self.files)+'\x00'
stringTableLen = len(stringTable)
stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00'
headerSizeNonPadded = 0x10 + len(self.files) * 0x18 + len(stringTableNonPadded)
stringTableSizePadded = len(stringTableNonPadded) + self.allign0x20(headerSizeNonPadded)
if self._stringTableSize == None:
self._stringTableSize = stringTableLen
if stringTableLen > self._stringTableSize:
self._stringTableSize = stringTableLen
self._stringTableSize = stringTableSizePadded
elif len(stringTableNonPadded) > self._stringTableSize:
self._stringTableSize = len(stringTableNonPadded)
return self._stringTableSize

def getFirstFileOffset(self):
return self.files[0].offset

def getHeader(self):
stringTable = '\x00'.join(file['name'] for file in self.files)
headerSize = 0x10 + len(self.files) * 0x18 + self.getStringTableSize()
stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00'
stringTableSizePadded = self.getStringTableSize()
stringTable = stringTableNonPadded + ('\x00'*(stringTableSizePadded-len(stringTableNonPadded)))
headerSize = 0x10 + len(self.files) * 0x18 + stringTableSizePadded

h = b''
h += b'PFS0'
h += len(self.files).to_bytes(4, byteorder='little')
h += (self.getStringTableSize()).to_bytes(4, byteorder='little')
h += (stringTableSizePadded).to_bytes(4, byteorder='little')
h += b'\x00\x00\x00\x00'

stringOffset = 0
Expand Down Expand Up @@ -140,27 +142,34 @@ def resize(self, name, size):
def close(self):
pass

#0xff => 0x1, 0x100 => 0x20, 0x1ff => 0x1, 0x120 => 0x20
def allign0x20(self, n):
return 0x20-n%0x20

def getStringTableSize(self):
stringTable = '\x00'.join(file['name'] for file in self.files)
stringTableLen = len(stringTable)
stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00'
headerSizeNonPadded = 0x10 + len(self.files) * 0x18 + len(stringTableNonPadded)
stringTableSizePadded = len(stringTableNonPadded) + self.allign0x20(headerSizeNonPadded)
if self._stringTableSize == None:
self._stringTableSize = stringTableLen
if stringTableLen > self._stringTableSize:
self._stringTableSize = stringTableLen
self._stringTableSize = stringTableSizePadded
elif len(stringTableNonPadded) > self._stringTableSize:
self._stringTableSize = len(stringTableNonPadded)
return self._stringTableSize

def getHash(self):
hexHash = self.binhash.hexdigest()
return hexHash

def getHeaderHash(self):
stringTable = '\x00'.join(file['name'] for file in self.files)
headerSize = 0x10 + len(self.files) * 0x18 + self.getStringTableSize()
stringTableNonPadded = '\x00'.join(file['name'] for file in self.files)+'\x00'
stringTableSizePadded = self.getStringTableSize()
stringTable = stringTableNonPadded + ('\x00'*(stringTableSizePadded-len(stringTableNonPadded)))
headerSize = 0x10 + len(self.files) * 0x18 + stringTableSizePadded

h = b''
h += b'PFS0'
h += len(self.files).to_bytes(4, byteorder='little')
h += (self.getStringTableSize()).to_bytes(4, byteorder='little')
h += (stringTableSizePadded).to_bytes(4, byteorder='little')
h += b'\x00\x00\x00\x00'

stringOffset = 0
Expand Down Expand Up @@ -191,6 +200,13 @@ def __init__(self, buffer, path = None, mode = None, cryptoType = -1, cryptoKey
#self.offset += sectionStart
#self.size -= sectionStart

#0xff => 0x1, 0x100 => 0x20, 0x1ff => 0x1, 0x120 => 0x20
def allign0x20(self, n):
return 0x20-n%0x20

def getPaddedHeaderSize(self):
return self._headerSize + self.allign0x20(self._headerSize);

def getHeaderSize(self):
return self._headerSize;

Expand Down
6 changes: 3 additions & 3 deletions nsz/NszDecompressor.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,17 +197,17 @@ def __decompressNsz(filePath, outputDir, removePadding, write, raiseVerification
filePathNsp = changeExtension(filePath, '.nsp')
outPath = filePathNsp if outputDir == None else str(Path(outputDir).joinpath(Path(filePathNsp).name))
Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint)
with Pfs0.Pfs0Stream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), outPath) as nsp:
with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), outPath) as nsp:
__decompressContainer(container, nsp, fileHashes, True, raiseVerificationException, raisePfs0Exception, statusReportInfo, pleaseNoPrint)
else:
with Pfs0.Pfs0VerifyStream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as nsp:
with Pfs0.Pfs0VerifyStream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as nsp:
__decompressContainer(container, nsp, fileHashes, True, raiseVerificationException, raisePfs0Exception, statusReportInfo, pleaseNoPrint)
Print.info("[PFS0 HEAD] " + nsp.getHeaderHash())
Print.info("[PFS0 DATA] " + nsp.getHash())
if originalFilePath != None:
originalContainer = factory(originalFilePath)
originalContainer.open(str(originalFilePath), 'rb')
with Pfs0.Pfs0VerifyStream(originalContainer.getHeaderSize() if removePadding else originalContainer.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as originalNsp:
with Pfs0.Pfs0VerifyStream(originalContainer.getPaddedHeaderSize() if removePadding else originalContainer.getFirstFileOffset(), None if removePadding else container.getStringTableSize()) as originalNsp:
__decompressContainer(originalContainer, originalNsp, fileHashes, True, raiseVerificationException, raisePfs0Exception, statusReportInfo, pleaseNoPrint)
Print.info("[PFS0 HEAD] " + originalNsp.getHeaderHash())
Print.info("[PFS0 DATA] " + originalNsp.getHash())
Expand Down
2 changes: 1 addition & 1 deletion nsz/SolidCompressor.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ def solidCompressNsp(filePath, compressionLevel, keepDelta, removePadding, useLo
Print.info(f'Solid compressing (level {compressionLevel}{" ldm" if useLongDistanceMode else ""}) {filePath} -> {nszPath}', pleaseNoPrint)

try:
with Pfs0.Pfs0Stream(container.getHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp:
with Pfs0.Pfs0Stream(container.getPaddedHeaderSize() if removePadding else container.getFirstFileOffset(), None if removePadding else container.getStringTableSize(), str(nszPath)) as nsp:
processContainer(container, nsp, compressionLevel, keepDelta, useLongDistanceMode, threads, statusReport, id, pleaseNoPrint)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Expand Down
3 changes: 3 additions & 0 deletions nsz/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,9 @@ def main():
if args.verify and not args.quick_verify and not args.keep_delta:
Print.info("Warning: --verify requires --keep-delta when used during compression or it will detect removed NDV0 fragments as errors. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.3.0.")
args.quick_verify = True
if args.verify and not args.quick_verify and args.remove_padding:
Print.info("Warning: --verify and --remove-padding are incompatible with each others. For compatibility reasons --quick-verify will be automatically used instead to match the command line argument behavior prior to NSZ v4.6.0.")
args.quick_verify = True
sourceFileToDelete = []
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
Expand Down