feat: Added the option to ignore files and folders via settings (#289)

* added rudimentary support for ignoring files * Added File Ignore Settings * Increased width of the ignore file settings textarea * fixed issue that occured when ignore settings were not initialized * fixed problem with loading settings if ignore settings don't exist * bumped version for release in niposch/obsidian_to_anki * Style.css and manifest are copied to build output folder * created prerelease 3.5.0-prerelease2 * handle undefined value when setting isn't initialized properly * reverted changes to rollup config * added missing dependency * reverted change to format.ts * added tests for ignore setting * Extended ignore_setting test to test if scandir and ignore setting work properly together * Added default glob for excalidraw; description in readme and plugin setting * fixed python tests * Squashed commit of the following: commit 867f230 Author: niposch <[email protected]> Date: Tue Jan 23 16:34:09 2024 +0100 fixed python tests * revert folder scan changes * fix ignore_setting.py * Update README.md * revert changes in test_folder_scan --------- Co-authored-by: Harsha Raghu <[email protected]>
ObsidianToAnki · Jan 23, 2024 · af2424d · af2424d
1 parent fd29403
commit af2424d
Show file tree

Hide file tree

Showing 16 changed files with 375 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -63,6 +63,16 @@ Current features (check out the wiki for more details):
 * **Custom scan directory** 
   * The plugin will scan the entire vault by default
   * You can also set which directory (includes all sub-directories as well) to scan via plugin settings
+* **Ignore Folders and Files**
+  * You can specify which files and folders to ignore 
+  * This can be done in the settings of this plugin with [Glob syntax](https://en.wikipedia.org/wiki/Glob_(programming)#Syntax).
+  * If you're working on your own globs, you can test them out [here](https://globster.xyz/)
+  * Examples:
+    * `**/*.excalidraw.md` - Ignore all files that end in `.excalidraw.md`
+      * => avoids excalidraw files from being scanned which can be extremely slow
+    * `Template/**` - Ignore all files in the `Template` folder (including subfolders)
+    * `**/private/**` - Ignore all files in folders that are called `private` no matter where they are in the vault
+    * `[Pp]rivate*/**` - Ignore all files and folders in the root of the vault that start with `private` or with `Private`
 * **Updating notes from file** - Your text files are the canonical source of the notes.
 * **Tags**, including **tags for an entire file**.
 * **Adding to user-specified deck** on a *per-file* basis.

diff --git a/main.ts b/main.ts
@@ -1,7 +1,7 @@
 import { Notice, Plugin, addIcon, TFile, TFolder } from 'obsidian'
 import * as AnkiConnect from './src/anki'
 import { PluginSettings, ParsedSettings } from './src/interfaces/settings-interface'
-import { SettingsTab } from './src/settings'
+import { DEFAULT_IGNORED_FILE_GLOBS, SettingsTab } from './src/settings'
 import { ANKI_ICON } from './src/constants'
 import { settingToData } from './src/setting-to-data'
 import { FileManager } from './src/files-manager'
@@ -42,7 +42,8 @@ export default class MyPlugin extends Plugin {
 				"CurlyCloze - Highlights to Clozes": false,
 				"ID Comments": true,
 				"Add Obsidian Tags": false,
-			}
+			},
+			IGNORED_FILE_GLOBS: DEFAULT_IGNORED_FILE_GLOBS,
 		}
 		/*Making settings from scratch, so need note types*/
 		this.note_types = await AnkiConnect.invoke('modelNames') as Array<string>

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -40,6 +40,7 @@
     },
     "dependencies": {
         "byte-base64": "^1.1.0",
+        "multimatch": "^7.0.0",
         "showdown": "^2.1.0",
         "ts-md5": "^1.2.7",
         "webdriver": "^8.5.5"

diff --git a/src/files-manager.ts b/src/files-manager.ts
@@ -4,7 +4,7 @@ import { App, TFile, TFolder, TAbstractFile, CachedMetadata, FileSystemAdapter,
 import { AllFile } from './file'
 import * as AnkiConnect from './anki'
 import { basename } from 'path'
-
+import multimatch from "multimatch"
 interface addNoteResponse {
     result: number,
     error: string | null
@@ -63,16 +63,25 @@ export class FileManager {
     constructor(app: App, data:ParsedSettings, files: TFile[], file_hashes: Record<string, string>, added_media: string[]) {
         this.app = app
         this.data = data
-        this.files = files
+
+        this.files = this.findFilesThatAreNotIgnored(files, data);
+
         this.ownFiles = []
         this.file_hashes = file_hashes
         this.added_media_set = new Set(added_media)
     }
-
     getUrl(file: TFile): string {
         return "obsidian://open?vault=" + encodeURIComponent(this.data.vault_name) + String.raw`&file=` + encodeURIComponent(file.path)
     }
 
+    findFilesThatAreNotIgnored(files:TFile[], data:ParsedSettings):TFile[]{
+        let ignoredFiles = []
+        ignoredFiles = multimatch(files.map(file => file.path), data.ignored_file_globs)
+
+        let notIgnoredFiles = files.filter(file => !ignoredFiles.contains(file.path))
+        return notIgnoredFiles;
+    }
+
     getFolderPathList(file: TFile): TFolder[] {
         let result: TFolder[] = []
         let abstractFile: TAbstractFile = file

diff --git a/src/interfaces/settings-interface.ts b/src/interfaces/settings-interface.ts
@@ -28,7 +28,8 @@ export interface PluginSettings {
 		"CurlyCloze - Highlights to Clozes": boolean,
 		"ID Comments": boolean,
 		"Add Obsidian Tags": boolean
-	}
+	},
+	IGNORED_FILE_GLOBS:string[]
 }
 
 export interface FileData {
@@ -59,4 +60,5 @@ export interface ParsedSettings extends FileData {
 	add_file_link: boolean
 	folder_decks: Record<string, string>
 	folder_tags: Record<string, string>
+	ignored_file_globs: string[]
 }
diff --git a/src/setting-to-data.ts b/src/setting-to-data.ts
@@ -42,6 +42,7 @@ export async function settingToData(app: App, settings: PluginSettings, fields_d
     result.comment = settings.Defaults["ID Comments"]
     result.add_context = settings.Defaults["Add Context"]
     result.add_obs_tags = settings.Defaults["Add Obsidian Tags"]
+    result.ignored_file_globs = settings.IGNORED_FILE_GLOBS ?? [];
 
     return result
 }
diff --git a/src/settings.ts b/src/settings.ts
@@ -14,6 +14,10 @@ const defaultDescs = {
 	"Add Obsidian Tags": "Interpret #tags in the fields of a note as Anki tags, removing them from the note text in Anki."
 }
 
+export const DEFAULT_IGNORED_FILE_GLOBS = [
+	'**/*.excalidraw.md'
+];
+
 export class SettingsTab extends PluginSettingTab {
 
 	setup_custom_regexp(note_type: string, row_cells: HTMLCollection) {
@@ -397,6 +401,35 @@ export class SettingsTab extends PluginSettingTab {
 				}
 			)
 	}
+	setup_ignore_files() {
+		let { containerEl } = this;
+		const plugin = (this as any).plugin
+		let ignored_files_settings = containerEl.createEl('h3', { text: 'Ignored File Settings' })
+		plugin.settings["IGNORED_FILE_GLOBS"] = plugin.settings.hasOwnProperty("IGNORED_FILE_GLOBS") ? plugin.settings["IGNORED_FILE_GLOBS"] : DEFAULT_IGNORED_FILE_GLOBS
+		const descriptionFragment = document.createDocumentFragment();
+		descriptionFragment.createEl("span", { text: "Glob patterns for files to ignore. You can add multiple patterns. One per line. Have a look at the " })
+		descriptionFragment.createEl("a", { text: "README.md", href: "https://github.com/Pseudonium/Obsidian_to_Anki?tab=readme-ov-file#features" });
+		descriptionFragment.createEl("span", { text: " for more information, examples and further resources." })
+
+
+		new Setting(ignored_files_settings)
+			.setName("Patterns to ignore")
+			.setDesc(descriptionFragment)
+			.addTextArea(text => {
+				text.setValue(plugin.settings.IGNORED_FILE_GLOBS.join("\n"))
+					.setPlaceholder("Examples: '**/*.excalidraw.md', 'Templates/**'")
+					.onChange((value) => {
+						let ignoreLines = value.split("\n")
+						ignoreLines = ignoreLines.filter(e => e.trim() != "") //filter out empty lines and blank lines
+						plugin.settings.IGNORED_FILE_GLOBS = ignoreLines
+
+						plugin.saveAllData()
+					}
+					)
+				text.inputEl.rows = 10
+				text.inputEl.cols = 30
+			});
+	}
 
 	setup_display() {
 		let {containerEl} = this
@@ -409,6 +442,7 @@ export class SettingsTab extends PluginSettingTab {
 		this.setup_syntax()
 		this.setup_defaults()
 		this.setup_buttons()
+		this.setup_ignore_files()
 	}
 
 	async display() {

diff --git a/tests/anki/test_ignore_setting.py b/tests/anki/test_ignore_setting.py
@@ -0,0 +1,82 @@
+import re
+import pytest
+from anki.errors import NotFoundError  # noqa
+from anki.collection import Collection
+from anki.collection import SearchNode
+
+# from conftest import col
+
+test_name = "ignore_setting"
+col_path = "tests/test_outputs/{}/Anki2/User 1/collection.anki2".format(test_name)
+
+test_file_paths = [
+    "tests/test_outputs/{}/Obsidian/{}/scan_dir/included_file.md".format(
+        test_name, test_name
+    ),
+    "tests/test_outputs/{}/Obsidian/{}/scan_dir/some/other/subdir/also_included_file.md".format(
+        test_name, test_name
+    ),
+]
+
+test_file_no_cards_paths = [
+    "tests/test_outputs/{}/Obsidian/{}/outside_of_scandir/not_supposed_to_be_scanned.md".format(
+        test_name, test_name
+    ),
+    "tests/test_outputs/{}/Obsidian/{}/scan_dir/ignored_by_setting_ignored/not_supposed_to_be_scanned.md".format(
+        test_name, test_name
+    ),
+    "tests/test_outputs/{}/Obsidian/{}/scan_dir/ignored_by_setting_ignored/some/other/subdir/not_supposed_to_be_scanned.md".format(
+        test_name, test_name
+    ),
+]
+
+
+@pytest.fixture()
+def col():
+    col = Collection(col_path)
+    yield col
+    col.close()
+
+
+def test_col_exists(col):
+    assert not col.is_empty()
+
+
+def test_deck_default_exists(col: Collection):
+    assert col.decks.id_for_name("Default") is not None
+
+
+def test_cards_count(col: Collection):
+    assert len(col.find_cards(col.build_search_string(SearchNode(deck="Default")))) == 6
+
+
+def test_cards_ids_from_obsidian(col: Collection):
+    ID_REGEXP_STR = r"\n?(?:<!--)?(?:ID: (\d+).*)"
+
+    obs_IDs = []
+    for obsidian_test_md in test_file_paths:
+        with open(obsidian_test_md) as file:
+            for line in file:
+                output = re.search(ID_REGEXP_STR, line.rstrip())
+                if output is not None:
+                    output = output.group(1)
+                    obs_IDs.append(output)
+
+    anki_IDs = col.find_notes(col.build_search_string(SearchNode(deck="Default")))
+    for aid, oid in zip(anki_IDs, obs_IDs):
+        assert str(aid) == oid
+
+
+def test_no_cards_added_from_ignored_paths(col: Collection):
+    ID_REGEXP_STR = r"\n?(?:<!--)?(?:ID: (\d+).*)"
+
+    for obsidian_test_md in test_file_no_cards_paths:
+        obs_IDs = []
+        with open(obsidian_test_md) as file:
+            for line in file:
+                output = re.search(ID_REGEXP_STR, line.rstrip())
+                if output is not None:
+                    output = output.group(1)
+                    obs_IDs.append(output)
+
+    assert len(obs_IDs) == 0
diff --git a/...ults/test_vault_suites/ignore_setting/.obsidian/plugins/obsidian-to-anki-plugin/data.json b/...ults/test_vault_suites/ignore_setting/.obsidian/plugins/obsidian-to-anki-plugin/data.json
@@ -0,0 +1,71 @@
+{
+  "settings": {
+    "CUSTOM_REGEXPS": {
+      "Basic": "",
+      "Basic (and reversed card)": "",
+      "Basic (optional reversed card)": "",
+      "Basic (type in the answer)": "",
+      "Cloze": ""
+    },
+    "FILE_LINK_FIELDS": {
+      "Basic": "Front",
+      "Basic (and reversed card)": "Front",
+      "Basic (optional reversed card)": "Front",
+      "Basic (type in the answer)": "Front",
+      "Cloze": "Text"
+    },
+    "CONTEXT_FIELDS": {},
+    "FOLDER_DECKS": {},
+    "FOLDER_TAGS": {},
+    "Syntax": {
+      "Begin Note": "START",
+      "End Note": "END",
+      "Begin Inline Note": "STARTI",
+      "End Inline Note": "ENDI",
+      "Target Deck Line": "TARGET DECK",
+      "File Tags Line": "FILE TAGS",
+      "Delete Note Line": "DELETE",
+      "Frozen Fields Line": "FROZEN"
+    },
+    "Defaults": {
+      "Scan Directory": "ignore_setting/scan_dir",
+      "Tag": "Obsidian_to_Anki",
+      "Deck": "Default",
+      "Scheduling Interval": 0,
+      "Add File Link": false,
+      "Add Context": false,
+      "CurlyCloze": true,
+      "CurlyCloze - Highlights to Clozes": false,
+      "ID Comments": true,
+      "Add Obsidian Tags": false
+    },
+    "IGNORED_FILE_GLOBS": [
+      "ignore_setting/scan_dir/*ignored/**"
+    ]
+  },
+  "Added Media": [],
+  "File Hashes": {},
+  "fields_dict": {
+    "Basic": [
+      "Front",
+      "Back"
+    ],
+    "Basic (and reversed card)": [
+      "Front",
+      "Back"
+    ],
+    "Basic (optional reversed card)": [
+      "Front",
+      "Back",
+      "Add Reverse"
+    ],
+    "Basic (type in the answer)": [
+      "Front",
+      "Back"
+    ],
+    "Cloze": [
+      "Text",
+      "Back Extra"
+    ]
+  }
+}
diff --git a/tests/defaults/test_vault_suites/ignore_setting/ignore_setting.md b/tests/defaults/test_vault_suites/ignore_setting/ignore_setting.md
@@ -0,0 +1 @@
+Just a file for the test to open