SQLite update

Docs/OneDriveExplorerManual.md

@@ -2,10 +2,11 @@
 
 ![](.\manual\ode.png)
 
-Revision history  
+**Revision history**  
 2022-11-08 Rev. 1 - Initial Release  
-2022-12-07 Rev. 2 - Updated for v2022.12.08
-2023-03-10 Rev. 3 - Updated for v2023.03.10
+2022-12-07 Rev. 2 - Updated for v2022.12.08  
+2023-03-10 Rev. 3 - Updated for v2023.03.10  
+2023-05-05 Rev. 4 - Updated for v2023.05.05  
 
 ## OneDriveExplorer GUI Introduction
 OneDriveExplorer GUI is used to view the contents of \<UserCid>.dat files. It can load multiple settings, logs, and $Recycle.bin files at once. Search across all settings files, view OneDrive logs and much more.
@@ -123,6 +124,17 @@ You can bypass these dialogs in one of two ways:
 
 ![](.\manual\preferences1.png)
 
+##### Load from SQLite
+Use this option to load OneDrive SQLite databases. Default location is *%USERPROFILE%\\%AppData%\Local\Microsoft\OneDrive\settings*. Once a folder is selected, OneDriveExplorer will prompt to load a User Hive.
+
+![](.\manual\userhive.png)
+
+You can bypass this dialog in one of two ways:
+* Holding down SHIFT when loading \<UserCid>.dat
+* In the Preferences dialog
+
+![](.\manual\preferences1.png)
+
 ##### Import JSON
 Import JSON allows for loading a previously saved JSON file from the command line or GUI application.
 
@@ -215,13 +227,14 @@ OneDriveExplorer is a tool used to parse \<UserCid\>.dat files and reconstruct t
 Running OneDriveExplorer.exe without any arguments displays a list of command line options:
 
 ```
-usage: OneDriveExplorer.py [-h] [-f FILE] [-d DIR] [-l [LOGS]] [-r REGHIVE] [-rb RECYCLE_BIN] [--csv CSV]
-                           [--csvf CSVF] [--html HTML] [--json JSON] [--pretty] [--clist] [--cstructs CSTRUCTS]
-                           [--sync] [--debug] [--guids]
+usage: OneDriveExplorer.exe [-h] [-f FILE] [-s SQL] [-d DIR] [-l [LOGS]] [-r REGHIVE] [-rb RECYCLE_BIN] [--csv CSV]
+                            [--csvf CSVF] [--html HTML] [--json JSON] [--pretty] [--clist] [--cstructs CSTRUCTS]
+                            [--sync] [--debug] [--guids]
 
 options:
   -h, --help            show this help message and exit
   -f FILE, --file FILE  <UserCid>.dat file to be parsed
+  -s SQL, --sql SQL     OneDrive folder containing SQLite databases
   -d DIR, --dir DIR     Directory to recursively process, looking for <UserCid>.dat, NTUSER hive, $Recycle.Bin, and
                         ODL logs. This mode is primarily used with KAPE.
   -l [LOGS], --logs [LOGS]
@@ -253,6 +266,8 @@ There are several groups of command line options for OneDriveExplorer.
 
 * **-f**: Full path, including \<UserCid>.dat, of \<UserCid>.dat file to be parse
 
+* **-s**: Directory containing OneDrive SQLite databases
+
 * **-d**: Directory to recursively process, looking for <UserCid>.dat, NTUSER hive, $Recycle.Bin, and ODL logs. This mode is primarily used with KAPE.
 
 * **-r**: This switch will instruct OneDriveExplorer to use the registry hive supplied to resolve OneDrive mount points.  
@@ -299,6 +314,10 @@ A user registry hive can be supplied with the `-r` argument. This will resolve s
 # Creating CStructs
 
 # Version changes
+## v2023.05.05
+### Added
+#### commandline/GUI
+* Added parsing of new OneDrive SQLite databases
 ## v2023.03.10
 ### Fixed 
 #### GUI

OneDriveExplorer/OneDriveExplorer.py

@@ -11,6 +11,7 @@
 from ode.parsers.dat import parse_dat
 from ode.parsers.onedrive import parse_onedrive
 from ode.parsers.odl import parse_odl, load_cparser
+from ode.parsers.sqlite_db import parse_sql
 from ode.utils import update_from_repo
 
 logging.basicConfig(level=logging.INFO,
@@ -19,7 +20,7 @@
                     )
 
 __author__ = "Brian Maloney"
-__version__ = "2023.03.10"
+__version__ = "2023.05.05"
 __email__ = "[email protected]"
 rbin = []
 
@@ -72,7 +73,7 @@ def output():
             logging.warning("KeyError: 'Folder'")
             folder_count = 0
 
-        print(f'{file_count} files(s) - {del_count} deleted, {folder_count} folder(s) in {format((time.time() - start), ".4f")} seconds')
+        print(f'\n\n{file_count} files(s) - {del_count} deleted, {folder_count} folder(s) in {format((time.time() - start), ".4f")} seconds\n')
 
     banner = r'''
      _____                ___                           ___                 _
@@ -89,6 +90,7 @@ def output():
     start = time.time()
     parser = argparse.ArgumentParser()
     parser.add_argument("-f", "--file", help="<UserCid>.dat file to be parsed")
+    parser.add_argument("-s", "--sql", help="OneDrive folder containing SQLite databases")
     parser.add_argument("-d", "--dir", help="Directory to recursively process, looking for <UserCid>.dat, NTUSER hive, $Recycle.Bin, and ODL logs. This mode is primarily used with KAPE.")
     parser.add_argument("-l", "--logs", help="Directory to recursively process for ODL logs.", nargs='?', const=True)
     parser.add_argument("-r", "--REG_HIVE", dest="reghive", help="If a registry hive is provided then the mount points of the SyncEngines will be resolved.")
@@ -123,7 +125,7 @@ def output():
         load_cparser(args.cstructs, args.clist)
         sys.exit()
 
-    if not args.file and not args.dir:
+    if not args.file and not args.dir and not args.sql:
         parser.print_help()
         print('\nEither -f or -d is required. Exiting')
         parser.exit()
@@ -160,6 +162,16 @@ def output():
                 print('Error: Remove trailing \ from directory.\nExample: --html "c:\\temp" ')
                 sys.exit()
 
+    if args.sql:
+        sql_dir = re.compile(r'\\Users\\(?P<user>.*?)\\AppData\\Local\\Microsoft\\OneDrive\\settings\\(?P<account>.*?)$')
+        sql_find = re.findall(sql_dir, args.sql)
+        try:
+            name = f'{sql_find[0][0]}_{sql_find[0][1]}'
+        except Exception:
+            name = 'SQLite_DB'
+        df, rbin_df = parse_sql(args.sql, args.reghive)
+        output()
+
     if args.file:
         account = os.path.dirname(args.file.replace('/', '\\')).rsplit('\\', 1)[-1]
         df, name = parse_dat(args.file, args.reghive, args.RECYCLE_BIN, start, account)
@@ -186,6 +198,7 @@ def output():
         d = {}
         hive = re.compile(r'\\Users\\(?P<user>.*)?')
         dat = re.compile(r'\\Users\\(?P<user>.*)?\\AppData\\Local\\Microsoft\\OneDrive\\settings')
+        sql_dir = re.compile(r'\\Users\\(?P<user>.*?)\\AppData\\Local\\Microsoft\\OneDrive\\settings\\(?P<account>.*?)$')
         log_dir = re.compile(r'\\Users\\(?P<user>.*)?\\AppData\\Local\\Microsoft\\OneDrive\\logs$')
         rootDir = args.dir
         spinner = spinning_cursor()
@@ -198,6 +211,7 @@ def output():
             hive_find = re.findall(hive, path)
             dat_find = re.findall(dat, path)
             log_find = re.findall(log_dir, path)
+            sql_find = re.findall(sql_dir, path)
             if path.endswith('$Recycle.Bin'):
                 args.RECYCLE_BIN = path
 
@@ -220,6 +234,10 @@ def output():
                 d.setdefault(log_find[0], {})
                 d[log_find[0]].setdefault('logs', []).append(path)
 
+            if sql_find:
+                d.setdefault(sql_find[0][0], {})
+                d[sql_find[0][0]].setdefault('sql', {})[f'{sql_find[0][1]}'] = path
+
         for key, value in d.items():
             filenames = []
             for k, v in value.items():
@@ -242,6 +260,18 @@ def output():
                         else:
                             output()
 
+                if k == 'sql':
+                    print(f'\n\nParsing {key} OneDrive\n')
+                    for account, sql_dir in v.items():
+                        name = f'{key}_{account}'
+                        df, rbin_df = parse_sql(sql_dir, args.reghive)
+                        if df.empty:
+                            print(f'Unable to parse {name} sqlite database.')
+                            logging.warning(f'Unable to parse {name} sqlite database.')
+                        else:
+                            output()
+
+
         if args.logs:
             load_cparser(args.cstructs)
             for key, value in d.items():

OneDriveExplorer/OneDriveExplorer_GUI.py

@@ -35,6 +35,7 @@
 from ode.parsers.csv_file import parse_csv
 from ode.parsers.onedrive import parse_onedrive
 from ode.parsers.odl import parse_odl, load_cparser
+from ode.parsers.sqlite_db import parse_sql
 from ode.helpers.mft import live_hive
 from ode.helpers import pandastablepatch
 from ode.helpers import ScrollableNotebookpatch
@@ -67,7 +68,7 @@
                     )
 
 __author__ = "Brian Maloney"
-__version__ = "2023.03.10"
+__version__ = "2023.05.05"
 __email__ = "[email protected]"
 rbin = []
 found = []
@@ -407,8 +408,9 @@ def close_pref(self):
 
 
 class hive:
-    def __init__(self, root):
+    def __init__(self, root, sql=False):
         self.root = root
+        self.sql = sql
         self.win = tk.Toplevel(self.root)
         self.win.wm_transient(self.root)
         self.win.title("Load User Hive")
@@ -475,7 +477,8 @@ def get_hive(self):
                                                          "*.dat"),))
         if reghive:
             self.win.destroy()
-            root.wait_window(rec_bin(root).win)
+            if not self.sql:
+                root.wait_window(rec_bin(root).win)
 
     def sync_windows(self, event=None):
         x = self.root.winfo_x()
@@ -1126,7 +1129,7 @@ def __init__(self, root):
                                 foreground='#0563C1', cursor="hand2",
                                 justify="left", anchor='w')
         self.text = tk.Text(self.frame, width=27, height=8, wrap=tk.WORD)
-        line = "GUI based application for reconstructing the folder structure of OneDrive from <UserCid>.dat"
+        line = "GUI based application for reconstructing the folder structure of OneDrive"
         self.text.insert(tk.END, line)
         self.text.config(state='disable')
         self.scrollbv = ttk.Scrollbar(self.frame, orient="vertical",
@@ -1860,6 +1863,7 @@ def live_system(menu):
     btn.configure(state="disabled")
     d = {}
     dat = re.compile(r'/Users\\(?P<user>.*)?\\AppData\\Local\\Microsoft\\OneDrive\\settings')
+    sql_dir = re.compile(r'/Users\\(?P<user>.*?)\\AppData\\Local\\Microsoft\\OneDrive\\settings\\(?P<account>.*?)$')
     log_dir = re.compile(r'/Users\\(?P<user>.*)?\\AppData\\Local\\Microsoft\\OneDrive\\logs$')
     rootDir = r'/'
     pb.configure(mode='indeterminate')
@@ -1868,25 +1872,32 @@ def live_system(menu):
     for path, subdirs, files in os.walk(rootDir):
         dat_find = re.findall(dat, path)
         log_find = re.findall(log_dir, path)
+        sql_find = re.findall(sql_dir, path)
         if path == '/$Recycle.Bin':
             recbin = path
+
         if dat_find:
             for name in files:
                 if '.dat' in name:
                     d.setdefault(dat_find[0], {})
                     d[dat_find[0]].setdefault('files', []).append(os.path.join(path, name))
+
         if log_find:
             d.setdefault(log_find[0], {})
             d[log_find[0]].setdefault('logs', []).append(path)
 
+        if sql_find:
+            d.setdefault(sql_find[0][0], {})
+            d[sql_find[0][0]].setdefault('sql', {})[f'{sql_find[0][1]}'] = path
+
     for key, value in d.items():
         filenames = []
         for k, v in value.items():
             if k == 'files':
                 filenames = v
 
             if len(filenames) != 0:
-                logging.info(f'Parsing OneDrive data for {key}')
+                logging.info(f'Parsing OneDrive dat for {key}')
                 menubar.entryconfig("File", state="disabled")
                 menubar.entryconfig("Options", state="disabled")
                 menubar.entryconfig("View", state="disabled")
@@ -1903,6 +1914,26 @@ def live_system(menu):
                     x = menu.entrycget(0, "label")
                     start_parsing(x, filename, reghive, recbin)
 
+            if k == 'sql':
+                logging.info(f'Parsing OneDrive SQLite for {key}')
+                menubar.entryconfig("File", state="disabled")
+                menubar.entryconfig("Options", state="disabled")
+                menubar.entryconfig("View", state="disabled")
+                menubar.entryconfig("Help", state="disabled")
+                search_entry.configure(state="disabled")
+                btn.configure(state="disabled")
+
+                if not reghive:
+                    value_label['text'] = f"Searching for {key}'s NTUSER.DAT. Please wait...."
+                    pb.configure(mode='indeterminate')
+                    pb.start()
+                    reghive = live_hive(key)
+                    pb.configure(mode='determinate')
+
+                for account, sql_dir in v.items():
+                    x = 'Load from SQLite'
+                    start_parsing(x, sql_dir, reghive)
+
     if menu_data['odl'] is True:
         menubar.entryconfig("File", state="disabled")
         menubar.entryconfig("Options", state="disabled")
@@ -1994,6 +2025,21 @@ def open_dat(menu):
     reghive = ''
     recbin = ''
 
+def read_sql(menu):
+    global reghive
+    folder_name = filedialog.askdirectory(initialdir="/", title="Open")
+
+    if folder_name:
+        if keyboard.is_pressed('shift') or menu_data['hive']:
+            pass
+        else:
+            root.wait_window(hive(root, sql=True).win)
+
+        x = menu.entrycget(1, "label")
+        threading.Thread(target=start_parsing,
+                         args=(x, folder_name, reghive,),
+                         daemon=True).start()
+    reghive = ''
 
 def import_json(menu):
     filename = filedialog.askopenfile(initialdir="/",
@@ -2002,7 +2048,7 @@ def import_json(menu):
                                                   "*.json"),))
 
     if filename:
-        x = menu.entrycget(1, "label")
+        x = menu.entrycget(2, "label")
         threading.Thread(target=start_parsing,
                          args=(x, filename,),
                          daemon=True).start()
@@ -2015,7 +2061,7 @@ def import_csv(menu):
                                                   "*.csv"),))
 
     if filename:
-        x = menu.entrycget(2, "label")
+        x = menu.entrycget(3, "label")
         threading.Thread(target=start_parsing,
                          args=(x, filename,),
                          daemon=True).start()
@@ -2195,6 +2241,18 @@ def start_parsing(x, filename=False, reghive=False, recbin=False, df=False):
                                      pb=pb, value_label=value_label)
         dat = True
 
+    if x == 'Load from SQLite':
+        filename = filename.replace('/', '\\')
+        sql_dir = re.compile(r'\\Users\\(?P<user>.*?)\\AppData\\Local\\Microsoft\\OneDrive\\settings\\(?P<account>.*?)$')
+        sql_find = re.findall(sql_dir, filename)
+        try:
+            name = f'{sql_find[0][0]}_{sql_find[0][1]}'
+        except Exception:
+            name = 'SQLite_DB'
+        df, rbin_df = parse_sql(filename, reghive)
+
+        dat = True
+
     if x == 'Import JSON':
         cache = json.load(filename)
         df = pd.DataFrame()
@@ -2948,6 +3006,7 @@ def sync():
 del_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/file_yellow_trashcan.png'))
 load_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/repeat_green.png'))
 live_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/computer_desktop.png'))
+sql_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/IDI_DB4S-1.png'))
 json_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/file_yellow_hierarchy1_expanded.png'))
 csv_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/table.png'))
 uaf_img = ImageTk.PhotoImage(Image.open(application_path + '/Images/delete_red.png'))
@@ -3148,6 +3207,9 @@ def sync():
                     image=load_img, compound='left',
                     command=lambda: open_dat(odsmenu),
                     accelerator="Ctrl+O")
+odsmenu.add_command(label="Load from SQLite",
+                    image=sql_img, compound='left',
+                    command=lambda: read_sql(odsmenu))
 odsmenu.add_command(label="Import JSON", image=json_img,
                     compound='left', command=lambda: import_json(odsmenu))
 odsmenu.add_command(label="Import CSV", image=csv_img, compound='left',