diff --git a/OneDriveExplorer/Images/splashv.png b/OneDriveExplorer/Images/splashv.png index 03ad4b5..15a50a6 100644 Binary files a/OneDriveExplorer/Images/splashv.png and b/OneDriveExplorer/Images/splashv.png differ diff --git a/OneDriveExplorer/OneDriveExplorer.py b/OneDriveExplorer/OneDriveExplorer.py index 5197efd..4815f00 100644 --- a/OneDriveExplorer/OneDriveExplorer.py +++ b/OneDriveExplorer/OneDriveExplorer.py @@ -31,6 +31,7 @@ import uuid import pandas as pd import warnings +import threading from ode.renderers.json import print_json from ode.renderers.csv_file import print_csv from ode.renderers.html import print_html @@ -50,12 +51,16 @@ ) __author__ = "Brian Maloney" -__version__ = "2024.11.01" +__version__ = "2024.11.12" __email__ = "bmmaloney97@gmail.com" rbin = [] DATParser = dat_parser.DATParser() OneDriveParser = onedrive_parser.OneDriveParser() SQLiteParser = sqlite_parser.SQLiteParser() +result = None +parsing_complete = threading.Event() +onedrive_complete = threading.Event() +output_complete = threading.Event() def spinning_cursor(): @@ -75,10 +80,36 @@ def guid(): break +def parse_sql_thread(sqlFolder): + global result + result = SQLiteParser.parse_sql(sqlFolder) + parsing_complete.set() # Signal that parsing is complete + + +def parse_onedrive_thread(df, df_scope, df_GraphMetadata_Records, scopeID, file, rbin_df, account, reghive, RECYCLE_BIN, localHashAlgorithm): + global result + result = None + result = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, file, rbin_df, account, reghive, RECYCLE_BIN, localHashAlgorithm) + onedrive_complete.set() # Signal that parsing is complete + + def main(): df_GraphMetadata_Records = pd.DataFrame(columns=['fileName', 'resourceID', 'graphMetadataJSON', 'spoCompositeID', 'createdBy', 'modifiedBy', 'filePolicies', 'fileExtension', 'lastWriteCount']) + def output_thread(): + delay = time.time() + + threading.Thread(target=output, + daemon=True).start() + + while not output_complete.is_set(): + if (time.time() - delay) > 0.1: + sys.stdout.write(f'Saving OneDrive data. Please wait.... {next(spinner)}\r') + sys.stdout.flush() + delay = time.time() + time.sleep(0.2) + def output(): if args.csv: print_csv(df, rbin_df, df_GraphMetadata_Records, name, args.csv, args.csvf) @@ -91,6 +122,8 @@ def output(): args.json = '.' print_json(cache, name, args.pretty, args.json) + output_complete.set() # Signal that parsing is complete + try: file_count = df.Type.value_counts()['File'] except KeyError: @@ -149,6 +182,8 @@ def output(): args = parser.parse_args() + spinner = spinning_cursor() + if args.sync: update_from_repo(args.gui) sys.exit() @@ -205,16 +240,44 @@ def output(): name = f'{sql_find[0][0]}_{sql_find[0][1]}' except Exception: name = 'SQLite_DB' - df, rbin_df, df_scope, df_GraphMetadata_Records, scopeID, account, localHashAlgorithm = SQLiteParser.parse_sql(args.sql) + + threading.Thread(target=parse_sql_thread, + args=(args.sql,), + daemon=True).start() + + delay = time.time() + while not parsing_complete.is_set(): + if (time.time() - delay) > 0.1: + sys.stdout.write(f'Parsing SQLite. Please wait.... {next(spinner)}\r') + sys.stdout.flush() + delay = time.time() + time.sleep(0.2) + + df, rbin_df, df_scope, df_GraphMetadata_Records, scopeID, account, localHashAlgorithm = result if not df.empty: - cache, rbin_df = OneDriveParser.parse_onedrive(df, df_scope, df_GraphMetadata_Records, scopeID, args.sql, rbin_df, account, args.reghive, args.RECYCLE_BIN, localHashAlgorithm) + threading.Thread(target=parse_onedrive_thread, + args=(df, df_scope, df_GraphMetadata_Records, + scopeID, args.sql, rbin_df, account, + args.reghive, args.RECYCLE_BIN, + localHashAlgorithm,), + daemon=True).start() + + delay = time.time() + while not onedrive_complete.is_set(): + if (time.time() - delay) > 0.1: + sys.stdout.write(f'Building folder list. Please wait.... {next(spinner)}\r') + sys.stdout.flush() + delay = time.time() + time.sleep(0.2) + + cache, rbin_df = result if df.empty: print(f'Unable to parse {name} sqlite database.') logging.warning(f'Unable to parse {name} sqlite database.') else: - output() + output_thread() rootDir = args.logs if rootDir is None: @@ -241,7 +304,7 @@ def output(): print(f'Unable to parse {filename}.') logging.warning(f'Unable to parse {filename}.') else: - output() + output_thread() rootDir = args.logs if rootDir is None: sys.exit() @@ -261,7 +324,7 @@ def output(): sql_dir = re.compile(r'\\Users\\(?P.*?)\\AppData\\Local\\Microsoft\\OneDrive\\settings\\(?PPersonal|Business[0-9])$') log_dir = re.compile(r'\\Users\\(?P.*)?\\AppData\\Local\\Microsoft\\OneDrive\\logs$') rootDir = args.dir - spinner = spinning_cursor() + # spinner = spinning_cursor() delay = time.time() for path, subdirs, files in os.walk(rootDir): if (time.time() - delay) > 0.1: @@ -323,7 +386,7 @@ def output(): print(f'Unable to parse {filename}.') logging.warning(f'Unable to parse {filename}.') else: - output() + output_thread() if k == 'sql': print(f'\n\nParsing {key} OneDrive\n') @@ -339,7 +402,7 @@ def output(): print(f'Unable to parse {name} sqlite database.') logging.warning(f'Unable to parse {name} sqlite database.') else: - output() + output_thread() if args.logs: load_cparser(args.cstructs) diff --git a/OneDriveExplorer/OneDriveExplorer_GUI.py b/OneDriveExplorer/OneDriveExplorer_GUI.py index 278b9c6..52108de 100644 --- a/OneDriveExplorer/OneDriveExplorer_GUI.py +++ b/OneDriveExplorer/OneDriveExplorer_GUI.py @@ -103,7 +103,7 @@ ) __author__ = "Brian Maloney" -__version__ = "2024.10.16" +__version__ = "2024.11.12" __email__ = "bmmaloney97@gmail.com" rbin = [] user_logs = {} @@ -333,7 +333,7 @@ def create_checkbuttons(self): self.auto_html.grid(row=2, column=0, columnspan=2, padx=5, sticky="w") self.reghive.grid(row=0, column=2, padx=5) self.en_odl.grid(row=0, column=0, padx=5, sticky="w") - self.en_cor.grid(row=1, column=0, padx=(15,5), sticky="w") + self.en_cor.grid(row=1, column=0, padx=(15, 5), sticky="w") self.auto_odl.grid(row=2, column=0, padx=5, sticky="w") def create_path_entry(self): @@ -608,7 +608,7 @@ def initialize_window(self): self.inner_frame = ttk.Frame(self.frame, relief='groove', padding=5) self.create_widgets() self.restore_tree_messages() - + self.sync_windows() self.root.bind('', self.sync_windows) @@ -1124,14 +1124,14 @@ def __init__(self, root): self.frame = ttk.Frame(self.win, relief='flat') self.create_text_widget() - + self.frame.grid(row=0, column=0) - + self.sync_windows() self.root.bind('', self.sync_windows) self.win.bind('', self.sync_windows) - + def configure_window(self): hwnd = get_parent(self.win.winfo_id()) old_style = get_window_long(hwnd, GWL_STYLE) @@ -1149,28 +1149,28 @@ def create_text_widget(self): "Live System\nRun OneDriveExplorer as an administrator to activate.", "For full details, see the included manual." ] - + # Calculate the height needed based on line count num_lines = sum(text.count("\n") + 1 for text in help_text) - + # Calculate the width based on the longest line max_line_length = max(len(subline) for line in help_text for subline in line.splitlines()) - + self.text_widget = tk.Text(self.frame, wrap="word", background=self.bgf, foreground=self.fgf, width=max_line_length, height=num_lines, relief='flat', state="normal") self.text_widget.bindtags((str(self.text_widget), str(self.root), "all")) self.text_widget.insert("1.0", "\n".join(help_text)) - + # Add a tag to the word "manual" self.text_widget.tag_add("manual_tag", "14.35", "14.41") # Assuming "manual" is at line 14, char 35-41 self.text_widget.tag_config("manual_tag", foreground='#0563C1', underline=True) - + bold_font = tkFont.Font(self.text_widget, self.text_widget.cget("font")) bold_font.configure(weight="bold") self.text_widget.tag_add("bold_tag", "1.0", "1.17") self.text_widget.tag_add("bold_tag", "7.0", "7.12") self.text_widget.tag_add("bold_tag", "12.0", "12.11") self.text_widget.tag_config("bold_tag", font=bold_font) - + # Bind double-click event to the "manual_tag" self.text_widget.tag_bind("manual_tag", "", self.open_manual) # Bind mouse enter and leave events to change cursor @@ -1205,7 +1205,7 @@ def __init__(self, root): self.create_window() self.configure_window() self.create_widgets() - + self.sync_windows() self.root.bind('', self.sync_windows) @@ -1446,7 +1446,7 @@ def handle_folder_status(self, num, values_list): (ast.literal_eval(item.split('spoPermissions: ')[1]) for item in self.args[0] if 'spoPermissions: ' in item), '' ) - + # Might need to look into this. #if num == '7' and len(values_list) > 12: if num == '7' and len(values_list) > 13: @@ -1469,7 +1469,7 @@ def handle_folder_status(self, num, values_list): ) if sharedItem == '1': - self.status.append(shared_big_img) + self.status.append(shared_big_img) if not set(self.lock_list).intersection(spoPermissions): if len(spoPermissions) > 0: @@ -1484,14 +1484,14 @@ def process_non_folder_status(self, values_list): '' ) sharedItem = next( - (item.split(' ')[1] for item in self.args[0] if 'shareditem:' in item.lower() and len(item.split(' ')) > 1), + (item.split(' ')[1] for item in self.args[0] if 'shareditem:' in item.lower() and len(item.split(' ')) > 1), '' ) hydrationType = next((item.split(' ')[1] for item in self.args[0] if 'lasthydrationtype:' in item.lower() and len(item.split(' ')) > 1), '') - + lastKnownPinState = next((item.split(' ')[1] for item in self.args[0] if 'lastknownpinstate:' in item.lower() and len(item.split(' ')) > 1), '') - + for num in ['2', '5', '6', '7', '8']: if any(('filestatus:' in item.lower() or 'inrecyclebin:' in item.lower()) and num in item for item in self.args[0]): if lastKnownPinState in ['0', '1'] and num == '2': @@ -1529,7 +1529,7 @@ def get_folder_color(self, num): '15': f_15_big_img } return folder_color[num] - + def get_type_image(self, num): type_dict = { '6': not_sync_big_img, # files @@ -1848,7 +1848,7 @@ def __init__(self, tv, parent, cur_sel, columns=('Date_created', 'Date_accessed' self.stop = threading.Event() self.status = [] self.file = False - + # Boolean variables for checkbuttons self.dateCreated = tk.BooleanVar(value=menu_data['Date_created']) self.dateAccessed = tk.BooleanVar(value=menu_data['Date_accessed']) @@ -1918,15 +1918,15 @@ def configure_columns(self): visableColumns.append("Date_modified") if self.size.get(): visableColumns.append("Size") - + menu_data['Date_created'] = self.dateCreated.get() menu_data['Date_accessed'] = self.dateAccessed.get() menu_data['Date_modified'] = self.dateModified.get() menu_data['Size'] = self.size.get() - + with open("ode.settings", "w") as jsonfile: json.dump(menu_data, jsonfile) - + self.tv3["displaycolumns"] = visableColumns def on_click(self, event): @@ -1965,10 +1965,10 @@ def sort_treeviews_mixed(self, tree1, tree2, col, descending): data = [(tree1.item(item, 'text'), item) for item in tree1.get_children('')] else: data = [(tree1.set(item, col), item) for item in tree1.get_children('')] - + # Get the tags for each item data_with_tags = [(self.tv3.item(item, 'tags')[0], row, item) for row, item in data] - + # Sort first by tag and then by the selected column if col == 'Size': sorted_data = sorted(data_with_tags, key=lambda x: (int(x[0]), self.extract_number(x[1])), reverse=descending) @@ -1984,13 +1984,13 @@ def sort_treeviews_mixed(self, tree1, tree2, col, descending): try: tree1.heading(col, command=lambda: self.sort_treeviews_mixed(tree1, tree2, col, not descending)) tree2.heading(col, command=lambda: self.sort_treeviews_mixed(tree1, tree2, col, not descending)) - except: + except Exception: pass def extract_number(self, s): match = re.search(r'\d+', s.replace(',', '')) return int(match.group()) if match else 0 - + def handle_click(self, event): if self.tv2.identify_region(event.x, event.y) == "separator": return "break" @@ -2104,7 +2104,7 @@ def select_item(self, event): in_recycle_bin = any('inrecyclebin' in value.lower() for value in values) # Determine the timestamp based on the conditions timestamp = "notificationTime: " if in_recycle_bin and values[1] == '' else ("DeleteTimeStamp: " if in_recycle_bin else "lastChange: ") - + if 'Date modified' in str(values[0]): values[0] = str(values[0])[17:].split("\n")[0] values[0] = f'{timestamp}{values[0]}' @@ -2119,7 +2119,6 @@ def select_item(self, event): try: for line in values: if line == '' or 'folderColor:' in line: - #if line == '': continue details.insert(tk.END, f'{line}\n', tags) except IndexError: @@ -2129,10 +2128,10 @@ def select_item(self, event): if len(values) > 4: details_frame.delete_tab(2) details_frame.delete_tab(1) - + # Check if 'fileStatus' or 'inRecycleBin' is in any value condition_met = any('fileStatus' in value or 'inRecycleBin' in value for value in values) - + if condition_met and not df_GraphMetadata_Records.empty: line_number = 4 start_index = f"{line_number}.0" @@ -2143,7 +2142,6 @@ def select_item(self, event): if not df_result.empty: tab2 = details_frame.add_tab('Metadata') self.meta_frame = details_frame.add_frame(tab2) - #self.meta_frame.configure(bg=bgf, padx=10, pady=10) self.get_resourceID(df_GraphMetadata_Records) def get_resourceID(self, df): @@ -2174,7 +2172,6 @@ def get_graphMetadataJSON(self, value): tab3 = details_frame.add_tab('MetadataJSON') self.json_frame = details_frame.add_frame(tab3) - #self.json_frame.configure(bg=bgf, padx=10, pady=10) row_num = 0 for k, v in value.items(): @@ -2204,7 +2201,6 @@ def get_filePolicies(self, value): tab4 = details_frame.add_tab('filePolicies') policy_frame = details_frame.add_frame(tab4) - #policy_frame.configure(bg=bgf, padx=10, pady=10) row_num = 0 @@ -2399,7 +2395,7 @@ def file_pane(self): self.tv2.insert("", "end", iid=child, image=image_key, text=text, values=values, tags=tags) folderStatus = next((item.split(' ')[1] for item in values if 'folderstatus:' in item.lower() and len(item.split(' ')) > 1), '') - + spoPermissions = next( (ast.literal_eval(item.split('spoPermissions: ')[1]) for item in values if 'spoPermissions: ' in item), '' @@ -2419,7 +2415,7 @@ def file_pane(self): ) if sharedItemF == '1': - self.status.append(shared_img) + self.status.append(shared_img) if not set(lock_list).intersection(spoPermissions) and str(tags) != 'red': if len(spoPermissions) > 0: @@ -2450,7 +2446,7 @@ def file_pane(self): if values_i[0] == '': # Find the item containing 'notificationtime:' and perform the operations notification_time_item = next((value for value in values_i if 'notificationtime:' in value.lower()), '') - + values_i[0] = notification_time_item[18:] index = values_i.index(notification_time_item) values_i[index] = 'DeleteTimeStamp: ' @@ -2463,7 +2459,7 @@ def file_pane(self): self.tv2.insert("", "end", iid=i, image=image_key_i, text=text_i, values=values_i, tags=tags_i) fileStatus = next((item.split(' ')[1] for item in values_i if ('filestatus:' in item.lower() or 'inrecyclebin' in item.lower()) and len(item.split(' ')) > 1), '') - + hydrationType = next((item.split(' ')[1] for item in values_i if 'lasthydrationtype:' in item.lower() and len(item.split(' ')) > 1), '') lastKnownPinState = next((item.split(' ')[1] for item in values_i if 'lastknownpinstate:' in item.lower() and len(item.split(' ')) > 1), '') @@ -2505,7 +2501,7 @@ def file_pane(self): cur_item = self.file self.file = False self.tv2.selection_set(cur_item) - + self.parent.update_idletasks() def insert_into_treeview(self, iid, image_sha1, values, tags): @@ -2513,7 +2509,7 @@ def insert_into_treeview(self, iid, image_sha1, values, tags): print('no tag') creationDate = next((item.split(' ', 1)[1] for item in values if 'diskcreationtime:' in item.lower() and len(item.split(' ', 1)) > 1), '') accessDate = next((item.split(' ', 1)[1] for item in values if 'disklastaccesstime:' in item.lower() and len(item.split(' ', 1)) > 1), '') - + new_values = [creationDate, accessDate, values[0], values[1]] if iid: self.tv3.insert("", "end", iid=iid, image=s_image[image_sha1], values=new_values, tags=tags) @@ -3030,7 +3026,7 @@ def add_frame(self, tab): self.frames.append(frame) self.text_boxes.append(text_box) return frame - + def update_textbox_theme(self, bg): # Update stored theme settings self.current_bg = bg @@ -3039,6 +3035,7 @@ def update_textbox_theme(self, bg): for text_box in self.text_boxes: text_box.configure(background=bg) + class CreateImage(): def __init__(self, status, type=False, small=False): self.status = status @@ -3046,7 +3043,7 @@ def __init__(self, status, type=False, small=False): self.small = small self.output_image = None self.sha1 = hashlib.sha1() - + self.create_output_image() self.sha1_digest = self.update_image_dictionary() @@ -3054,12 +3051,14 @@ def create_output_image(self): # Calculate total width for the output image if self.type: total_width = 84 + elif self.small: + total_width = 16 else: total_width = sum(img.width for img in self.status) + (33 if self.type else 0) - + # Determine the height for the output image (32 or 16) output_height = 32 if self.type else 16 - + # Create a new RGBA image with the appropriate size self.output_image = Image.new("RGBA", (total_width, output_height), (0, 0, 0, 0)) # If 'type' exists, paste it at the beginning @@ -3082,12 +3081,11 @@ def update_image_dictionary(self): self.output_image.save(fp, 'png') fp.seek(0) self.sha1.update(fp.read()) - #image = ImageTk.PhotoImage(Image.open(fp)) digest = self.sha1.hexdigest() if digest not in s_image: image = ImageTk.PhotoImage(Image.open(fp)) s_image[digest] = image - + return digest @@ -3330,7 +3328,7 @@ def pane_config(): pass try: file_manager.update_theme() - except: + except Exception: pass ttk.Style().theme_use() @@ -3430,7 +3428,7 @@ def search_result(): def clear_search(): global s_image -# s_image.clear() + position = None threading.Thread(target=clear_tvr, daemon=True).start() @@ -3624,12 +3622,6 @@ def parent_child(d, parent_id=None, meta=False): if 'Scope' in d: for c in d['Scope']: - image = link_directory_img - if c['shortcutItemIndex'] > 0: - image = vault_closed_img - if c['siteID'] == '' and '+' in c['scopeID']: - image = sync_directory_img - x = ('', '') y = [f'{k}: {v}' if v is not None else f'{k}: ' for k, v in c.items() if 'Links' not in k] @@ -3637,8 +3629,33 @@ def parent_child(d, parent_id=None, meta=False): w = [f'{k}: {v}' for k, v in b.items() if 'Files' not in k and 'Folders' not in k] z = x + tuple(w) + tuple(y) + if c['shortcutItemIndex'] > 0: + image = vault_closed_img + else: + if 'folderColor' in b and b['folderColor'] in range(1, 16): + folder_type.append(folder_color.get(b['folderColor'])) + else: + folder_type.append(dir_img) + + if c['siteID'] == '' and '+' in c['scopeID']: + if b['folderStatus'] == 9: + folder_type.append(sync_img) + elif b['folderStatus'] == 10: + folder_type.append(not_sync_img) + elif b['folderStatus'] == 11: + folder_type.append(not_link_img) + elif b['folderStatus'] == 12: + folder_type.append(unknown_img) + else: + folder_type.append(link_img) + + image_creator = CreateImage(folder_type, False, True) + image_sha1 = image_creator.sha1_digest + image = s_image[image_sha1] + if ('Folders' in b or 'Files' in b) and str(image) == str(vault_closed_img): image = vault_open_img + parent_child(b, tv.insert(parent_id, 0, image=image, @@ -3886,6 +3903,7 @@ def odl(folder_name, csv=False): if csv: key = folder_name.name.split('/')[-1].split('_')[0] header_list = ['Profile', + 'Key_Type', 'Log_Type', 'Filename', 'File_Index', @@ -3895,6 +3913,7 @@ def odl(folder_name, csv=False): 'Code_File', 'Flags', 'Function', + 'Context_Data', 'Description', 'Params', 'Param1', @@ -4507,6 +4526,7 @@ def widgets_disable(): details_frame.delete_tab(1) tv.grid_forget() + def widgets_normal(): tabs = tb.tabs() for i, item in enumerate(tabs): @@ -4524,15 +4544,17 @@ def widgets_normal(): btn.configure(state="normal") tv.grid(row=1, column=0, sticky="nsew") + def get_total_column_width(): total_width = file_manager.tv3.column("#0", width=None) total_width += file_manager.tv2.column("#0", width=None) for col in file_manager.tv3["columns"]: - if menu_data[str(col)] == True: + if menu_data[str(col)]: total_width += file_manager.tv3.column(col, width=None) # Get the current width of each column return total_width + # Function to limit sash movement def restrict_sash(*args): if dragging_sash: @@ -4544,12 +4566,14 @@ def restrict_sash(*args): pwh.sash_place(2, max_sash, 1) # Restrict sash to column width pwh.after(1, restrict_sash) + # Function to start checking when the drag begins def start_drag(event): global dragging_sash dragging_sash = True restrict_sash() + # Function to stop checking when the drag ends def stop_drag(event): global dragging_sash @@ -4714,6 +4738,7 @@ def stop_drag(event): dir_img = Image.open(application_path + '/Images/folders/directory.png') directory_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/folders/directory_closed.png')) link_directory_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/folders/67.png')) +sync_directory_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/folders/sync_directory.png')) # small folder images color f_1_img = Image.open(application_path + '/Images/colors/1.png') diff --git a/OneDriveExplorer/ode/helpers/Manual/OneDriveExplorerManual.html b/OneDriveExplorer/ode/helpers/Manual/OneDriveExplorerManual.html index c9d4e0b..a2b664f 100644 --- a/OneDriveExplorer/ode/helpers/Manual/OneDriveExplorerManual.html +++ b/OneDriveExplorer/ode/helpers/Manual/OneDriveExplorerManual.html @@ -612,7 +612,7 @@

Navigation Pane

File/Folder Pane

The file/folder pane shows the contents of the folder selected in the navigation pane along with it's OneDrive status. Once a file/folder has been loaded and a folder has been selected, a context menu is available by right clicking on the file/folder. Context menu options will be discussed later.

Status
-

File folder statu is a s follows:
+

File folder status is a s follows:

*Note: Not Synced and Not Linked do not exist on the endpoint. These are artifacts of syncing and linking libraries.

Details

diff --git a/OneDriveExplorer/ode/parsers/odl.py b/OneDriveExplorer/ode/parsers/odl.py index c7ef1ee..f879949 100644 --- a/OneDriveExplorer/ode/parsers/odl.py +++ b/OneDriveExplorer/ode/parsers/odl.py @@ -52,13 +52,14 @@ cparser = '' # UnObfuscation code -dkey_list = [] +dkey_list = {} +key_type = '' utf_type = 'utf16' headers = ''' typedef struct _Odl_header{ - char signature[8]; // EBFGONED + uint64 signature; // 0x44454e4f47464245 EBFGONED uint32 odl_version; } Odl_header; @@ -79,7 +80,9 @@ } Odl_header_V2_3; typedef struct _Data_block_V2{ - uint64 signature; // CCDDEEFF00000000 + uint32 signature; // CCDDEEFF + uint16 context_data_len; + uint16 unknown_flag; uint64 timestamp; uint32 unk1; uint32 unk2; @@ -92,12 +95,15 @@ } Data_block_V2; typedef struct _Data_block_V3{ - uint64 signature; // CCDDEEFF00000000 + uint32 signature; // CCDDEEFF + uint16 context_data_len; + uint16 unknown_flag; uint64 timestamp; uint32 unk1; uint32 unk2; uint32 data_len; uint32 unk3; + char context_data[context_data_len]; // followed by Data } Data_block_V3; @@ -249,24 +255,27 @@ def decrypt(cipher_text): if len(cipher_text) % 16 != 0: return '' # invalid b64 or it was not encrypted! - for key in dkey_list: - try: - cipher = AES.new(key, AES.MODE_CBC, iv=b'\0'*16) - raw = cipher.decrypt(cipher_text) - except ValueError as ex: - # log.error(f'Exception while decrypting data {str(ex)}') - return '' - try: - plain_text = unpad(raw, 16) - except Exception as ex: # possible fix to change key - # print("Error in unpad!", str(ex), raw) - continue - try: - plain_text = plain_text.decode(utf_type) - except ValueError as ex: - # print(f"Error decoding {utf_type}", str(ex)) - return '' - return plain_text + for key, values in dkey_list.items(): + global key_type + key_type = key + for value in values: + try: + cipher = AES.new(value, AES.MODE_CBC, iv=b'\0'*16) + raw = cipher.decrypt(cipher_text) + except ValueError as ex: + # log.error(f'Exception while decrypting data {str(ex)}') + return '' + try: + plain_text = unpad(raw, 16) + except Exception as ex: # possible fix to change value + # print("Error in unpad!", str(ex), raw) + continue + try: + plain_text = plain_text.decode(utf_type) + except ValueError as ex: + # print(f"Error decoding {utf_type}", str(ex)) + return '' + return plain_text def read_keystore(keystore_path): @@ -280,8 +289,8 @@ def read_keystore(keystore_path): version = j[0]['Version'] utf_type = 'utf32' if dkey.endswith('\\u0000\\u0000') else 'utf16' log.info(f"Recovered Unobfuscation key from {f.name}, key:{dkey}, version:{version}, utf_type:{utf_type}") - if base64.b64decode(dkey) not in dkey_list: - dkey_list.append(base64.b64decode(dkey)) + if not any(base64.b64decode(dkey) in values for values in dkey_list.values()): + dkey_list.setdefault(os.path.basename(f.name).split('.')[0], []).append(base64.b64decode(dkey)) if version != 1: log.warning(f'WARNING: Key version {version} is unsupported. This may not work. Contact the author if you see this to add support for this version.') except ValueError as ex: @@ -375,6 +384,31 @@ def extract_strings(data, map): return extracted +def extract_context_data(data): + hex_str = data.hex() + index = 0 + extracted_text = '' + + length_hex = hex_str[2:6] + length_hex = length_hex[2:4] + length_hex[0:2] + length = int(length_hex, 16) + index += 6 + + extracted_text += f"{bytes.fromhex(hex_str[index:index + length * 2]).decode('utf-8', errors='ignore')}" + index += length * 2 + + while index < len(hex_str): + segment_length_hex = hex_str[index:index + 2] + segment_length = int(segment_length_hex, 16) + index += 4 + + text_segment = bytes.fromhex(hex_str[index:index + segment_length * 2]).decode('utf-8', errors='ignore') + extracted_text += f' {text_segment}' + index += segment_length * 2 + + return extracted_text + + def unobfucate_strings(data, map): params = [] exclude = ['_len', 'unk'] @@ -391,6 +425,7 @@ def unobfucate_strings(data, map): def process_odl(filename, map): + global key_type odl_rows = [] basename = os.path.basename(filename) profile = os.path.dirname(filename).split('\\')[-1] @@ -412,7 +447,8 @@ def process_odl(filename, map): except Exception: log.warning(f'Unable to parse {basename}. Not a valid log file.') return pd.DataFrame() - if header.signature == b'EBFGONED': # Odl header + + if header.signature == 0x44454e4f47464245: # Odl header EBFGONED pass else: log.error(f'{basename} wrong header! Did not find EBFGONED') @@ -431,13 +467,14 @@ def process_odl(filename, map): f.close() f = io.BytesIO(file_data) signature = f.read(8) - if signature != b'\xCC\xDD\xEE\xFF\0\0\0\0': # CDEF header + if signature[0:4] != b'\xCC\xDD\xEE\xFF': # CDEF header log.error(f'{basename} wrong header! Did not find 0xCCDDEEFF') return pd.DataFrame() else: + context_data_len = int.from_bytes(signature[4:6], byteorder='little') f.seek(-8, 1) if header.odl_version == 3: - db_size = 32 + db_size = 32 + context_data_len else: db_size = 56 data_block = f.read(db_size) # odl complete header is 56 bytes @@ -445,6 +482,7 @@ def process_odl(filename, map): description = '' odl = { 'Profile': profile, + 'Key_Type': key_type, 'Log_Type': basename.split('-')[0], 'Filename': basename, 'File_Index': i, @@ -454,6 +492,7 @@ def process_odl(filename, map): 'Code_File': '', 'Flags': '', 'Function': '', + 'Context_Data': '', 'Description': '', 'Params': '', 'Param1': '', @@ -470,25 +509,33 @@ def process_odl(filename, map): 'Param12': '', 'Param13': '' } + if header.odl_version in [1, 2]: data_block = cparser.Data_block_V2(data_block) elif header.odl_version == 3: data_block = cparser.Data_block_V3(data_block) else: log.error(f'Unknown odl_version = {header.odl_version}') + if data_block.signature != 0xffeeddcc: log.warning(f'Unable to parse {basename} completely. Did not find 0xCCDDEEFF') return pd.DataFrame.from_records(odl_rows) + timestamp = ReadUnixMsTime(data_block.timestamp) odl['Timestamp'] = timestamp + try: if header.odl_version == 3: - data = cparser.Data_v3(f.read(data_block.data_len)) - params_len = (data_block.data_len - data.code_file_name_len - data.code_function_name_len - 36) + if data_block.context_data_len > 0: + data = cparser.Data_v2(f.read(data_block.data_len - data_block.context_data_len)) + params_len = (data_block.data_len - data_block.context_data_len - data.code_file_name_len - data.code_function_name_len - 12) + else: + data = cparser.Data_v3(f.read(data_block.data_len)) + params_len = (data_block.data_len - data.code_file_name_len - data.code_function_name_len - 36) else: data = cparser.Data_v2(f.read(data_block.data_len)) params_len = (data_block.data_len - data.code_file_name_len - data.code_function_name_len - 12) - f.seek(- params_len, io.SEEK_CUR) + f.seek(- params_len, 1) except Exception as e: log.warning(f'Unable to parse {basename} completely. {type(e).__name__}') return pd.DataFrame.from_records(odl_rows) @@ -519,7 +566,9 @@ def process_odl(filename, map): odl['Param13'] = params[12] except Exception: pass + params = ', '.join(params) + description = ''.join([v for (k, v) in cparser.consts.items() if k == f"{data.code_file_name.decode('utf8').lower().split('.')[0]}_{data.flags}_{data.code_function_name.decode('utf8').split('::')[-1].replace('~', '_').replace(' ()', '_').lower()}_des"]) except EOFError: log.warning(f"EOFError while parsing {data.code_file_name.decode('utf8').lower().split('.')[0]}_{data.flags}_{data.code_function_name.decode('utf8').split('::')[-1].replace('~', '_').replace(' ()', '_').lower()}") @@ -530,14 +579,27 @@ def process_odl(filename, map): else: params = '' + + odl['Key_Type'] = key_type odl['Code_File'] = data.code_file_name.decode('utf8') odl['Flags'] = data.flags odl['Function'] = data.code_function_name.decode('utf8') + odl['Context_Data'] = extract_context_data(data_block.context_data) if data_block.context_data else '' odl['Description'] = description odl['Params'] = params odl_rows.append(odl) + + key_type = '' i += 1 - data_block = f.read(db_size) + eof = f.read(8)[4:6] + + if eof == b'': + break + + context_data_len = int.from_bytes(eof, byteorder='little') + f.seek(-8, 1) + data_block = f.read(db_size + context_data_len) + return pd.DataFrame.from_records(odl_rows) diff --git a/OneDriveExplorer/ode/parsers/onedrive.py b/OneDriveExplorer/ode/parsers/onedrive.py index 0c9e87c..124e923 100644 --- a/OneDriveExplorer/ode/parsers/onedrive.py +++ b/OneDriveExplorer/ode/parsers/onedrive.py @@ -90,9 +90,9 @@ def find_parent(self, x, id_name_dict, parent_dict): # Generate scopeID list instead of passing def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_path, rbin_df, account=False, reghive=False, recbin=False, localHashAlgorithm=False, gui=False, pb=False, value_label=False): - + allowed_keys = ['scopeID', 'siteID', 'webID', 'listID', 'tenantID', 'webURL', 'remotePath', 'MountPoint', 'spoPermissions', 'shortcutVolumeID', 'shortcutItemIndex'] - + df_scope['shortcutVolumeID'] = df_scope['shortcutVolumeID'].apply(lambda x: '{:08x}'.format(x) if pd.notna(x) else '') df_scope['shortcutVolumeID'] = df_scope['shortcutVolumeID'].apply(lambda x: '{}{}{}{}-{}{}{}{}'.format(*x.upper()) if x else '') @@ -190,14 +190,11 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p cache = {} final = [] - dcache = {} is_del = [] if not df_GraphMetadata_Records.empty: df_GraphMetadata_Records.set_index('resourceID', inplace=True) - column_len = len(df.columns) - for row in df.sort_values( by=['Level', 'parentResourceID', 'Type', 'FileSort', 'FolderSort', 'libraryType'], ascending=[False, False, False, True, False, False]).to_dict('records'): @@ -302,7 +299,7 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p df_GraphMetadata_Records.reset_index(inplace=True) try: df_GraphMetadata_Records.drop('index', axis=1, inplace=True) - except: + except Exception: pass return cache, rbin_df diff --git a/OneDriveExplorer/ode/parsers/sqlite_db.py b/OneDriveExplorer/ode/parsers/sqlite_db.py index 4cc521a..8ef6dbf 100644 --- a/OneDriveExplorer/ode/parsers/sqlite_db.py +++ b/OneDriveExplorer/ode/parsers/sqlite_db.py @@ -28,7 +28,6 @@ import sqlite3 from dissect import cstruct import pandas as pd -import numpy as np from ode.utils import permissions, change_dtype @@ -153,7 +152,7 @@ def parse_sql(self, sql_dir): df_files['spoPermissions'] = df_files['spoPermissions'].apply(lambda x: permissions(x)) df_files['lastChange'] = pd.to_datetime(df_files['lastChange'], unit='s').astype(str) - if 23 < schema_version <= 32: + if 23 < schema_version: df_folders = pd.read_sql_query("SELECT parentScopeID, parentResourceID, resourceID, eTag, folderName, folderStatus, spoPermissions, volumeID, itemIndex, folderColor, sharedItem FROM od_ClientFolder_Records", SyncEngineDatabase) else: df_folders = pd.read_sql_query("SELECT parentScopeID, parentResourceID, resourceID, eTag, folderName, folderStatus, spoPermissions, volumeID, itemIndex, sharedItem FROM od_ClientFolder_Records", SyncEngineDatabase) @@ -164,11 +163,11 @@ def parse_sql(self, sql_dir): df = pd.concat([df_scope, df_files, df_folders], ignore_index=True, axis=0) df = df.where(pd.notnull(df), None) - + if df.empty: self.log.warning(f'{sql_dir}\SyncEngineDatabase.db is empty.') - if schema_version >=10: + if schema_version >= 10: df_GraphMetadata_Records = pd.read_sql_query("SELECT fileName, od_GraphMetadata_Records.* FROM od_GraphMetadata_Records INNER JOIN od_ClientFile_Records ON od_ClientFile_Records.resourceID = od_GraphMetadata_Records.resourceID", SyncEngineDatabase) df_GraphMetadata_Records = change_dtype(df_GraphMetadata_Records, df_name='df_GraphMetadata_Records', schema_version=schema_version) if not df_GraphMetadata_Records.empty: diff --git a/OneDriveExplorer/ode/renderers/project.py b/OneDriveExplorer/ode/renderers/project.py index c5aa5da..5bbcfc0 100644 --- a/OneDriveExplorer/ode/renderers/project.py +++ b/OneDriveExplorer/ode/renderers/project.py @@ -44,7 +44,7 @@ def load_images(zip_name): filenames = archive.namelist() filtered_list = [item for item in filenames if item.startswith('Images/')] sorted_list = sorted(filtered_list) - + for img in sorted_list: with archive.open(img) as data: digest = str(img).split('_')[1].split('.png')[0] @@ -52,7 +52,7 @@ def load_images(zip_name): s_image[digest] = image return s_image - + except Exception as e: log.error(f'Error loading images from {zip_name.split("/")[-1]}. {e}') @@ -179,7 +179,7 @@ def get_total_items(tree, item): with open(tmp_file, 'rb') as image_file: archive.writestr(f'Images/{index}_{k}.png', image_file.read()) os.remove(tmp_file) - + for i in d: filename = f"{tv.item(i)['text'].split('.')[0][1:]}_OneDrive.csv" diff --git a/OneDriveExplorer/ode/utils.py b/OneDriveExplorer/ode/utils.py index 53e5cf9..0092379 100644 --- a/OneDriveExplorer/ode/utils.py +++ b/OneDriveExplorer/ode/utils.py @@ -220,7 +220,7 @@ def change_dtype(df, df_name=None, schema_version=0): dtype_fill_map['df_scope']['fill_values']['webURL'] = '' dtype_fill_map['df_scope']['dtype_changes']['remotePath'] = 'str' dtype_fill_map['df_scope']['fill_values']['remotePath'] = '' - + if df_name == 'df_GraphMetadata_Records' and schema_version > 13: dtype_fill_map['df_GraphMetadata_Records']['dtype_changes']['filePolicies'] = 'str' dtype_fill_map['df_GraphMetadata_Records']['fill_values']['filePolicies'] = '' @@ -228,7 +228,7 @@ def change_dtype(df, df_name=None, schema_version=0): dtype_fill_map['df_GraphMetadata_Records']['fill_values']['fileExtension'] = '' dtype_fill_map['df_GraphMetadata_Records']['dtype_changes']['lastWriteCount'] = 'Int64' dtype_fill_map['df_GraphMetadata_Records']['fill_values']['lastWriteCount'] = 0 - + # Apply changes if df_name is recognized if df_name in dtype_fill_map: df.fillna(dtype_fill_map[df_name]['fill_values'], inplace=True)