Add misc scripts for posterity (mostly useless)

scripts/pcsx2_trace_process.py

@@ -0,0 +1,125 @@
+# Author: https://github.com/VelocityRa
+
+import sys
+import os
+from pathlib import Path
+import struct
+import io
+import operator
+
+tracefile_path = Path("D:\\Nikos\\Programming\\cpp\\pcsx2\\pcsx2\\windows\\VCprojects\\pcsx2_trace2.bin")
+coveragefile_path = Path("D:\\Nikos\\Programming\\cpp\\pcsx2\\pcsx2\\windows\\VCprojects\\pcsx2_trace2.trace")
+exe_path = Path("D:\\Roms\\PS2\\Games\\Sly Cooper and the Thievius Raccoonus (USA)\\SCUS_971.98")
+exe_text_offset = 0x1000
+exe_text_size = 0x17B294
+module_base = 0x00100000
+exclude_vaddr_list = [ 0x9488 ]
+count_hits = True
+trace_data = {}
+
+def is_inst_that_splits_block(instruction):
+    opcode = instruction >> 26
+
+    splits_standard = [2, 3, 4, 5, 6, 7, 20, 21, 22, 23]
+    if opcode in splits_standard:
+        return True
+
+    if opcode == 0: # SPECIAL
+        opcode = instruction & 0x3F
+
+        splits_special = [8, 9]
+
+        if opcode in splits_special:
+            return True
+
+    if opcode == 1: # REGIMM
+        opcode = (instruction >> 16) & 0x1F
+
+        splits_regimm = [0, 1, 2, 3, 16, 17, 18, 19]
+
+        if opcode in splits_regimm:
+            return True
+
+    return False
+
+
+with open(tracefile_path , 'rb') as trace_f:
+    trace_f.seek(0, io.SEEK_END)
+    entry_filesize = trace_f.tell()
+    trace_f.seek(0, io.SEEK_SET)
+
+    entry_count = entry_filesize // 4 - len(exclude_vaddr_list)
+
+    with open(coveragefile_path , 'wb') as cov_f:
+        cov_f.write(b"DDPH-PINTOOL\n")
+        cov_f.write(bytes("EntryCount: {}, ModuleCount: {}\n".format(entry_count, 1), encoding='utf8'))
+        cov_f.write(b"Module table row names (left to right): Module Id,  Module Base, Module End, Module Path\n\n")
+
+        cov_f.write(b"MODULE_TABLE\n")
+        cov_f.write(bytes("0, {:X}, {:X}, none\n".format(module_base, module_base + exe_text_size), encoding='utf8'))
+
+        cov_f.write(b"ENTRY_TABLE\n")
+
+        max_range_size = 0
+        with open(exe_path, 'rb') as exe_f:
+            for i in range(entry_count):
+                # if i==392:
+                #     what=4
+
+                # Write range start offset
+
+                pc = struct.unpack('I', trace_f.read(4))[0]
+                if pc < module_base or pc >= module_base + exe_text_size:
+                    continue
+
+                pc_rel = pc - module_base
+
+                if pc_rel in exclude_vaddr_list:
+                    continue
+
+                if count_hits:
+                    hits = struct.unpack('I', trace_f.read(4))[0]
+                    trace_data[pc_rel] = hits
+
+                cov_f.write(struct.pack('I', pc_rel))
+
+                # Write range size
+
+                # Walk instructions till we find one that would split the current block (
+                range_size = 4 + 4 # +4 for branch delay slot
+
+                exe_f.seek(exe_text_offset + pc_rel)
+                while True:
+                    instruction = int.from_bytes(exe_f.read(4), byteorder='little')
+
+                    # if pc_rel == 0x9488 and range_size // 4 ==  14:
+                    #     print("ins: {:X}".format(instruction))
+                    #     what=1
+
+                    if is_inst_that_splits_block(instruction):
+                        break
+
+                    range_size += 4
+
+                cov_f.write(struct.pack('H', range_size))
+
+                # Write module id
+                cov_f.write(b'\x00\x00')
+
+                # Write instruction count
+                inst_count = range_size // 4
+                cov_f.write(struct.pack('I', inst_count))
+
+                # print("Wrote start=0x{:X} icount=0x{:X}".format(pc_rel, inst_count))
+
+                max_range_size = max(range_size, max_range_size)
+
+            print("Done. entry count: {} max icount: 0x{:X}".format(entry_count, max_range_size//4))
+
+
+trace_data = dict(sorted(trace_data.items(), key=operator.itemgetter(1), reverse=True))
+
+if count_hits:
+    print("Addr Hits")
+    for k, v in trace_data:
+        print("{:08X} {}".format(k, v))

scripts/sly1_extract.py

@@ -0,0 +1,114 @@
+# Author: https://github.com/VelocityRa
+
+import sys
+import os
+from pathlib import Path
+# from mmap import ACCESS_READ, mmap
+# from DXTDecompress import DXTBuffer
+# from PIL import Image
+import struct
+import io
+
+# Config
+EXTRACT_WAC_CONTENTS = True
+PRINT_WAC_ENTRIES = False
+WAC_NAME = "SLY.WAC"
+#
+
+if len(sys.argv) < 1:
+    print("Enter the dir to SLY.WAC/WAL as command-line argument")
+
+base_path = Path(sys.argv[1])
+
+wal_path = base_path / "SLY.WAL"
+wac_path = base_path / WAC_NAME
+out_dir_path = base_path / "{}_contents".format(WAC_NAME)
+
+out_dir_path.mkdir(exist_ok=True)
+
+class WACEntry:
+    def __init__(self, name, type_, offset, size):
+        self.name = name
+        self.type_ = type_
+        self.offset = offset
+        self.size = size
+
+    def __str__(self):
+        return '{:08X} {:08X} {:08X} {} {}'.format(self.offset, self.offset*2048, self.size, self.type_, self.name)
+
+
+wac_entries = []
+
+with open(wac_path, 'rb') as wac_f:
+    wac_entry_count = struct.unpack("i", wac_f.read(4))[0]
+
+    # File has 2 extra bytes at the end for some reason
+    for i in range(wac_entry_count):
+        name, type_, offset, size = struct.unpack("23sBii", wac_f.read(0x20))
+
+        # Strip out null chars at the end - nothing else worked
+        name_stripped = ""
+        for n in name:
+            if n == 0:
+                break
+            name_stripped += chr(n)
+
+        wac_entries.append(WACEntry(name_stripped, chr(type_), offset, size))
+
+def print_wac_entries():
+    print("Offset(sector) Offset(bytes) Size Type Name")
+    for wac_entry in wac_entries:
+        print(wac_entry)
+
+if PRINT_WAC_ENTRIES:
+    print_wac_entries()
+
+SECTOR_SIZE = 2048
+
+def lookup_sector(sector_to_find):
+    for wac_entry in wac_entries:
+        start = wac_entry.offset
+        end = wac_entry.offset + (wac_entry.size // SECTOR_SIZE)
+
+        if start <= sector_to_find < end:
+            print("Found sector 0x{:X} in [{}], offset 0x{:X} sectors".format(sector_to_find, wac_entry,
+                                                                              sector_to_find - wac_entry.offset))
+
+def lookup_byte(byte_to_find):
+    for wac_entry in wac_entries:
+        start = wac_entry.offset
+        end = (wac_entry.offset * SECTOR_SIZE) + wac_entry.size
+
+        if start <= byte_to_find < end:
+            print("Found byte 0x{:X} at [{}], offset 0x{:X} bytes".format(byte_to_find, wac_entry,
+                                                                          byte_to_find - wac_entry.offset * SECTOR_SIZE))
+
+# lookup_sector(820372 - 0x3AFCE)
+
+def extract_wac_contents():
+    # with open(wal_path, 'rb') as wal_f, mmap(wal_f.fileno(), 0, access=ACCESS_READ) as wal_data:
+    with open(wal_path, 'rb') as wal_f:
+        for wac_entry in wac_entries:
+            wac_offset = wac_entry.offset * SECTOR_SIZE
+
+            extension = "bin"
+
+            wal_f.seek(wac_offset)
+
+            if wal_f.read(4) == b"VAGp":
+                extension = "vag"
+            else:
+                wal_f.seek(wac_offset + 0x20)
+
+                if wal_f.read(4) == b"klBS":
+                    extension = "bnk"
+
+            filename = "{}_{}_0x{:X}_0x{:X}.{}".format(wac_entry.name, wac_entry.type_, wac_entry.offset, wac_entry.size, extension)
+            out_dir_path_final = out_dir_path / filename # TODO
+
+            with open(out_dir_path_final, 'wb') as out_f:
+                wal_f.seek(wac_offset)
+                out_f.write(wal_f.read(wac_entry.size))
+
+if EXTRACT_WAC_CONTENTS:
+    extract_wac_contents()

scripts/sly1_iso_search.py

@@ -0,0 +1,56 @@
+# Author: https://github.com/VelocityRa
+
+import os
+from pathlib import Path
+
+# from mmap import ACCESS_READ, mmap
+# from DXTDecompress import DXTBuffer
+# from PIL import Image
+# import array
+
+iso_path = Path("D:\\Roms\\PS2\\Games\\Sly\\SLY.mdf")
+
+SECTOR_SIZE = 0x800
+
+# data = np.fromfile(iso_path, 'uint8')
+
+ff_byte_indices = []
+
+with open(iso_path, "rb") as iso_f:
+    filesize = os.path.getsize(iso_path)
+
+    # data = array.array('B')
+    # data.fromfile(iso_f, filesize // data.itemsize)
+
+    for i in range(0, filesize, SECTOR_SIZE):
+        iso_f.seek(i)
+
+        if iso_f.read(1)[0] == 0xFF:
+            iso_f.seek(i)
+            data_preview = iso_f.read(16)
+            data_preview_str = ''.join('%02x' % i for i in data_preview)
+            print("0xFF at sector {} (byte 0x{:X})  {}".format(
+                i // SECTOR_SIZE, i, data_preview_str))
+
+            ff_byte_indices.append(i)
+
+    ff_byte_indices_sec = []
+    for i in ff_byte_indices:
+        ff_byte_indices_sec.append(i // SECTOR_SIZE)
+
+    ff_byte_indices_abs = []
+    for i in ff_byte_indices:
+        ff_byte_indices_abs.append(i | 0x20000000)
+
+    ff_byte_indices.append(ff_byte_indices_abs)
+
+    for j in range(0, filesize, 4):
+        iso_f.seek(j)
+
+        word = int.from_bytes(iso_f.read(4), byteorder='little')
+
+        if word in ff_byte_indices:
+            print("\tFound1 seg for offset {:08X} at 0x{:08X}".format(i, j))
+
+        if word in ff_byte_indices_sec:
+            print("\tFound2 seg for offset {:08X} at 0x{:08X}".format(i, j))