feat: Add new option to set custom scan directory (#154)

* Add custom scan dir * Update test suites data.json with new option * Account for new setting * Fix undefined variable * Replace spaces with tabs in changed files Seems like source code uses tabs (for most part), so replacing spaces with tabs in files that I've changed for the PR * Add e2e tests * Update filename to fix spec requirement * Add subdir to scan and fix headings in md files * Update readme with new feature description * fix tests --------- Co-authored-by: Harsha Raghu <[email protected]>
ObsidianToAnki · Oct 5, 2023 · 3839220 · 3839220
1 parent 72becdf
commit 3839220
Show file tree

Hide file tree

Showing 24 changed files with 328 additions and 5 deletions.
diff --git a/README.md b/README.md
@@ -60,6 +60,9 @@ The script needs to be able to:
 
 Current features (check out the wiki for more details):
 * **Custom note types** - You're not limited to the 6 built-in note types of Anki.
+* **Custom scan directory** 
+  * The plugin will scan the entire vault by default
+  * You can also set which directory (includes all sub-directories as well) to scan via plugin settings
 * **Updating notes from file** - Your text files are the canonical source of the notes.
 * **Tags**, including **tags for an entire file**.
 * **Adding to user-specified deck** on a *per-file* basis.

diff --git a/main.ts b/main.ts
@@ -1,4 +1,4 @@
-import { Notice, Plugin, addIcon } from 'obsidian'
+import { Notice, Plugin, addIcon, TFile, TFolder } from 'obsidian'
 import * as AnkiConnect from './src/anki'
 import { PluginSettings, ParsedSettings } from './src/interfaces/settings-interface'
 import { SettingsTab } from './src/settings'
@@ -32,6 +32,7 @@ export default class MyPlugin extends Plugin {
 				"Frozen Fields Line": "FROZEN"
 			},
 			Defaults: {
+				"Scan Directory": "",
 				"Tag": "Obsidian_to_Anki",
 				"Deck": "Default",
 				"Scheduling Interval": 0,
@@ -49,8 +50,8 @@ export default class MyPlugin extends Plugin {
 		for (let note_type of this.note_types) {
 			settings["CUSTOM_REGEXPS"][note_type] = ""
 			const field_names: string[] = await AnkiConnect.invoke(
-	            'modelFieldNames', {modelName: note_type}
-	        ) as string[]
+				'modelFieldNames', {modelName: note_type}
+			) as string[]
 			this.fields_dict[note_type] = field_names
 			settings["FILE_LINK_FIELDS"][note_type] = field_names[0]
 		}
@@ -155,6 +156,32 @@ export default class MyPlugin extends Plugin {
 		}
 	}
 
+	/**
+	 * Recursively traverse a TFolder and return all TFiles.
+	 * @param tfolder - The TFolder to start the traversal from.
+	 * @returns An array of TFiles found within the folder and its subfolders.
+	 */
+	getAllTFilesInFolder(tfolder) {
+		const allTFiles = [];
+		// Check if the provided object is a TFolder
+		if (!(tfolder instanceof TFolder)) {
+			return allTFiles;
+		}
+		// Iterate through the contents of the folder
+		tfolder.children.forEach((child) => {
+			// If it's a TFile, add it to the result
+			if (child instanceof TFile) {
+				allTFiles.push(child);
+			} else if (child instanceof TFolder) {
+				// If it's a TFolder, recursively call the function on it
+				const filesInSubfolder = this.getAllTFilesInFolder(child);
+				allTFiles.push(...filesInSubfolder);
+			}
+			// Ignore other types of files or objects
+		});
+		return allTFiles;
+	}
+
 	async scanVault() {
 		new Notice('Scanning vault, check console for details...');
 		console.info("Checking connection to Anki...")
@@ -167,7 +194,22 @@ export default class MyPlugin extends Plugin {
 		}
 		new Notice("Successfully connected to Anki! This could take a few minutes - please don't close Anki until the plugin is finished")
 		const data: ParsedSettings = await settingToData(this.app, this.settings, this.fields_dict)
-		const manager = new FileManager(this.app, data, this.app.vault.getMarkdownFiles(), this.file_hashes, this.added_media)
+		const scanDir = this.app.vault.getAbstractFileByPath(this.settings.Defaults["Scan Directory"])
+		let manager = null;
+		if (scanDir !== null) {
+			let markdownFiles = [];
+			if (scanDir instanceof TFolder) {
+				console.info("Using custom scan directory: " + scanDir.path)
+				markdownFiles = this.getAllTFilesInFolder(scanDir);
+			} else {
+				new Notice("Error: incorrect path for scan directory " + this.settings.Defaults["Scan Directory"])
+				return
+			}
+			manager = new FileManager(this.app, data, markdownFiles, this.file_hashes, this.added_media)
+		} else {
+			manager = new FileManager(this.app, data, this.app.vault.getMarkdownFiles(), this.file_hashes, this.added_media);
+		}
+
 		await manager.initialiseFiles()
 		await manager.requests_1()
 		this.added_media = Array.from(manager.added_media_set)

diff --git a/src/interfaces/settings-interface.ts b/src/interfaces/settings-interface.ts
@@ -18,6 +18,7 @@ export interface PluginSettings {
 		"Frozen Fields Line": string
 	},
 	Defaults: {
+		"Scan Directory": string,
 		"Tag": string,
 		"Deck": string,
 		"Scheduling Interval": number
@@ -55,7 +56,7 @@ export interface FileData {
 }
 
 export interface ParsedSettings extends FileData {
-    add_file_link: boolean
+	add_file_link: boolean
 	folder_decks: Record<string, string>
 	folder_tags: Record<string, string>
 }
diff --git a/src/settings.ts b/src/settings.ts
@@ -2,6 +2,7 @@ import { PluginSettingTab, Setting, Notice, TFolder } from 'obsidian'
 import * as AnkiConnect from './anki'
 
 const defaultDescs = {
+	"Scan Directory": "The directory to scan. Leave empty to scan the entire vault",
 	"Tag": "The tag that the plugin automatically adds to any generated cards.",
 	"Deck": "The deck the plugin adds cards to if TARGET DECK is not specified in the file.",
 	"Scheduling Interval": "The time, in minutes, between automatic scans of the vault. Set this to 0 to disable automatic scanning.",
@@ -170,6 +171,10 @@ export class SettingsTab extends PluginSettingTab {
 		const plugin = (this as any).plugin
 		let defaults_settings = containerEl.createEl('h3', {text: 'Defaults'})
 
+		// To account for new scan directory
+		if (!(plugin.settings["Defaults"].hasOwnProperty("Scan Directory"))) {
+			plugin.settings["Defaults"]["Scan Directory"] = ""
+		}
 		// To account for new add context
 		if (!(plugin.settings["Defaults"].hasOwnProperty("Add Context"))) {
 			plugin.settings["Defaults"]["Add Context"] = false

diff --git a/tests/anki/test_folder_scan.py b/tests/anki/test_folder_scan.py
@@ -0,0 +1,117 @@
+
+import re
+import pytest
+from anki.errors import NotFoundError  # noqa
+from anki.collection import Collection
+from anki.collection import SearchNode
+# from conftest import col
+
+test_name = 'folder_scan'
+col_path = 'tests/test_outputs/{}/Anki2/User 1/collection.anki2'.format(test_name)
+
+test_file_paths = [
+    'tests/test_outputs/{}/Obsidian/{}/scan_dir/{}.md'.format(test_name, test_name, test_name),
+    'tests/test_outputs/{}/Obsidian/{}/scan_dir/also_scan/folder_scan_subdir.md'.format(test_name, test_name),
+]
+
+test_file_no_cards_paths = [
+    'tests/test_outputs/{}/Obsidian/{}/should_not_scan_dir/should_not_scan.md'.format(test_name, test_name),
+    'tests/test_outputs/{}/Obsidian/{}/{}.md'.format(test_name, test_name, test_name),
+]
+
+@pytest.fixture()
+def col():
+    col = Collection(col_path)
+    yield col
+    col.close()
+
+def test_col_exists(col):
+    assert not col.is_empty()
+
+def test_deck_default_exists(col: Collection):
+    assert col.decks.id_for_name('Default') is not None
+
+def test_cards_count(col: Collection):
+    assert len(col.find_cards( col.build_search_string(SearchNode(deck='Default')) )) == 8
+
+def test_cards_ids_from_obsidian(col: Collection):
+
+    ID_REGEXP_STR = r'\n?(?:<!--)?(?:ID: (\d+).*)'
+
+    for obsidian_test_md in test_file_paths:
+        obs_IDs = []
+        with open(obsidian_test_md) as file:
+            for line in file:            
+                output = re.search(ID_REGEXP_STR, line.rstrip())
+                if output is not None:
+                    output = output.group(1)
+                    obs_IDs.append(output)
+
+    anki_IDs = col.find_notes( col.build_search_string(SearchNode(deck='Default')) )
+    for aid, oid in zip(anki_IDs, obs_IDs):
+        assert str(aid) == oid
+
+def test_no_cards_added_from_obsidian(col: Collection):
+
+    ID_REGEXP_STR = r'\n?(?:<!--)?(?:ID: (\d+).*)'
+
+    for obsidian_test_md in test_file_no_cards_paths:
+        obs_IDs = []
+        with open(obsidian_test_md) as file:
+            for line in file:            
+                output = re.search(ID_REGEXP_STR, line.rstrip())
+                if output is not None:
+                    output = output.group(1)
+                    obs_IDs.append(output)
+
+    assert len(obs_IDs) == 0
+
+def test_cards_front_back_tag_type(col: Collection):
+
+    anki_IDs = col.find_notes( col.build_search_string(SearchNode(deck='Default')) )
+
+    # assert that should_not_add_notes weren't added
+    assert len(anki_IDs) == 8
+
+    note1 = col.get_note(anki_IDs[0])
+    assert note1.fields[0] == "Style Subdir"
+    assert note1.fields[1] == "This style is suitable for having the header as the front, and the answer as the back"
+
+    note2 = col.get_note(anki_IDs[1])
+    assert note2.fields[0] == "Subheading 1 Subdir"
+    assert note2.fields[1] == "You're allowed to nest headers within each other"
+
+    note3 = col.get_note(anki_IDs[2])
+    assert note3.fields[0] == "Subheading 2 Subdir"
+    assert note3.fields[1] == "It'll take the deepest level for the question"
+
+    note4 = col.get_note(anki_IDs[3])
+    assert note4.fields[0] == "Subheading 3 Subdir"
+    assert note4.fields[1] == "It'll even<br />\nSpan over<br />\nMultiple lines, and ignore preceding whitespace"
+
+
+    note5 = col.get_note(anki_IDs[4])
+    assert note5.fields[0] == "Style"
+    assert note5.fields[1] == "This style is suitable for having the header as the front, and the answer as the back"
+
+    note6 = col.get_note(anki_IDs[5])
+    assert note6.fields[0] == "Subheading 1"
+    assert note6.fields[1] == "You're allowed to nest headers within each other"
+
+    note7 = col.get_note(anki_IDs[6])
+    assert note7.fields[0] == "Subheading 2"
+    assert note7.fields[1] == "It'll take the deepest level for the question"
+
+    note8 = col.get_note(anki_IDs[7])
+    assert note8.fields[0] == "Subheading 3"
+    assert note8.fields[1] == "It'll even<br />\nSpan over<br />\nMultiple lines, and ignore preceding whitespace"
+
+    assert note1.note_type()["name"] == "Basic"
+    assert note2.note_type()["name"] == "Basic"
+    assert note3.note_type()["name"] == "Basic"
+    assert note4.note_type()["name"] == "Basic"
+
+    assert note5.note_type()["name"] == "Basic"
+    assert note6.note_type()["name"] == "Basic"
+    assert note7.note_type()["name"] == "Basic"
+    assert note8.note_type()["name"] == "Basic"
diff --git a/...defaults/test_vault_suites/basic_para/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...defaults/test_vault_suites/basic_para/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -28,6 +28,7 @@
       "Frozen Fields Line": "FROZEN"
     },
     "Defaults": {
+      "Scan Directory": "",
       "Tag": "Obsidian_to_Anki",
       "Deck": "Default",
       "Scheduling Interval": 0,

diff --git a/...faults/test_vault_suites/basic_para_3/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...faults/test_vault_suites/basic_para_3/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -28,6 +28,7 @@
       "Frozen Fields Line": "FROZEN"
     },
     "Defaults": {
+      "Scan Directory": "",
       "Tag": "Obsidian_to_Anki",
       "Deck": "Default",
       "Scheduling Interval": 0,

diff --git a/...lts/test_vault_suites/cloze_highlight/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...lts/test_vault_suites/cloze_highlight/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -28,6 +28,7 @@
       "Frozen Fields Line": "FROZEN"
     },
     "Defaults": {
+      "Scan Directory": "",
       "Tag": "Obsidian_to_Anki",
       "Deck": "Default",
       "Scheduling Interval": 0,

diff --git a/...defaults/test_vault_suites/cloze_para/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...defaults/test_vault_suites/cloze_para/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -28,6 +28,7 @@
       "Frozen Fields Line": "FROZEN"
     },
     "Defaults": {
+      "Scan Directory": "",
       "Tag": "Obsidian_to_Anki",
       "Deck": "Default",
       "Scheduling Interval": 0,

diff --git a/...defaults/test_vault_suites/cloze_sync/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...defaults/test_vault_suites/cloze_sync/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -28,6 +28,7 @@
       "Frozen Fields Line": "FROZEN"
     },
     "Defaults": {
+      "Scan Directory": "",
       "Tag": "Obsidian_to_Anki",
       "Deck": "Default",
       "Scheduling Interval": 0,

diff --git a/...faults/test_vault_suites/context_test/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...faults/test_vault_suites/context_test/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -30,6 +30,7 @@
       "Frozen Fields Line": "FROZEN"
     },
     "Defaults": {
+      "Scan Directory": "",
       "Tag": "Obsidian_to_Anki",
       "Deck": "Default",
       "Scheduling Interval": 0,

diff --git a/...efaults/test_vault_suites/folder_deck/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...efaults/test_vault_suites/folder_deck/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -32,6 +32,7 @@
       "Frozen Fields Line": "FROZEN"
     },
     "Defaults": {
+      "Scan Directory": "",
       "Tag": "Obsidian_to_Anki",
       "Deck": "Default",
       "Scheduling Interval": 0,

diff --git a/...ts/test_vault_suites/folder_deck_tags/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...ts/test_vault_suites/folder_deck_tags/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -36,6 +36,7 @@
       "Frozen Fields Line": "FROZEN"
     },
     "Defaults": {
+      "Scan Directory": "",
       "Tag": "Obsidian_to_Anki",
       "Deck": "Default",
       "Scheduling Interval": 0,

diff --git a/...efaults/test_vault_suites/folder_scan/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...efaults/test_vault_suites/folder_scan/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -0,0 +1,68 @@
+{
+  "settings": {
+    "CUSTOM_REGEXPS": {
+      "Basic": "^#+(.+)\n*((?:\n(?:^[^\n#].{0,2}$|^[^\n#].{3}(?<!<!--).*))+)",
+      "Basic (and reversed card)": "",
+      "Basic (optional reversed card)": "",
+      "Basic (type in the answer)": "",
+      "Cloze": ""
+    },
+    "FILE_LINK_FIELDS": {
+      "Basic": "Front",
+      "Basic (and reversed card)": "Front",
+      "Basic (optional reversed card)": "Front",
+      "Basic (type in the answer)": "Front",
+      "Cloze": "Text"
+    },
+    "CONTEXT_FIELDS": {},
+    "FOLDER_DECKS": {},
+    "FOLDER_TAGS": {},
+    "Syntax": {
+      "Begin Note": "START",
+      "End Note": "END",
+      "Begin Inline Note": "STARTI",
+      "End Inline Note": "ENDI",
+      "Target Deck Line": "TARGET DECK",
+      "File Tags Line": "FILE TAGS",
+      "Delete Note Line": "DELETE",
+      "Frozen Fields Line": "FROZEN"
+    },
+    "Defaults": {
+      "Scan Directory": "folder_scan/scan_dir",
+      "Tag": "Obsidian_to_Anki",
+      "Deck": "Default",
+      "Scheduling Interval": 0,
+      "Add File Link": false,
+      "Add Context": false,
+      "CurlyCloze": true,
+      "CurlyCloze - Highlights to Clozes": false,
+      "ID Comments": true,
+      "Add Obsidian Tags": false
+    }
+  },
+  "Added Media": [],
+  "File Hashes": {},
+  "fields_dict": {
+    "Basic": [
+      "Front",
+      "Back"
+    ],
+    "Basic (and reversed card)": [
+      "Front",
+      "Back"
+    ],
+    "Basic (optional reversed card)": [
+      "Front",
+      "Back",
+      "Add Reverse"
+    ],
+    "Basic (type in the answer)": [
+      "Front",
+      "Back"
+    ],
+    "Cloze": [
+      "Text",
+      "Back Extra"
+    ]
+  }
+}
diff --git a/tests/defaults/test_vault_suites/folder_scan/folder_scan.md b/tests/defaults/test_vault_suites/folder_scan/folder_scan.md
@@ -0,0 +1,17 @@
+# Style fsmd
+This style is suitable for having the header as the front, and the answer as the back
+# Overall heading fsmd
+
+## Subheading 1 fsmd
+You're allowed to nest headers within each other
+
+## Subheading 2 fsmd
+It'll take the deepest level for the question
+
+## Subheading 3 fsmd
+
+
+
+It'll even
+Span over
+Multiple lines, and ignore preceding whitespace