From af2424d47409d59c5396e52bf8bd0d0efdc9815b Mon Sep 17 00:00:00 2001 From: niposch Date: Tue, 23 Jan 2024 22:45:53 +0100 Subject: [PATCH] feat: Added the option to ignore files and folders via settings (#289) * added rudimentary support for ignoring files * Added File Ignore Settings * Increased width of the ignore file settings textarea * fixed issue that occured when ignore settings were not initialized * fixed problem with loading settings if ignore settings don't exist * bumped version for release in niposch/obsidian_to_anki * Style.css and manifest are copied to build output folder * created prerelease 3.5.0-prerelease2 * handle undefined value when setting isn't initialized properly * reverted changes to rollup config * added missing dependency * reverted change to format.ts * added tests for ignore setting * Extended ignore_setting test to test if scandir and ignore setting work properly together * Added default glob for excalidraw; description in readme and plugin setting * fixed python tests * Squashed commit of the following: commit 867f2300731830bc98eb3562a6eae8d2ed240b4d Author: niposch Date: Tue Jan 23 16:34:09 2024 +0100 fixed python tests * revert folder scan changes * fix ignore_setting.py * Update README.md * revert changes in test_folder_scan --------- Co-authored-by: Harsha Raghu --- README.md | 10 +++ main.ts | 5 +- package-lock.json | 39 +++++++++ package.json | 1 + src/files-manager.ts | 15 +++- src/interfaces/settings-interface.ts | 4 +- src/setting-to-data.ts | 1 + src/settings.ts | 34 ++++++++ tests/anki/test_ignore_setting.py | 82 +++++++++++++++++++ .../plugins/obsidian-to-anki-plugin/data.json | 71 ++++++++++++++++ .../ignore_setting/ignore_setting.md | 1 + .../not_supposed_to_be_scanned.md | 24 ++++++ .../not_supposed_to_be_scanned.md | 23 ++++++ .../subdir/not_supposed_to_be_scanned.md | 23 ++++++ .../ignore_setting/scan_dir/included_file.md | 24 ++++++ .../some/other/subdir/also_included_file.md | 24 ++++++ 16 files changed, 375 insertions(+), 6 deletions(-) create mode 100644 tests/anki/test_ignore_setting.py create mode 100644 tests/defaults/test_vault_suites/ignore_setting/.obsidian/plugins/obsidian-to-anki-plugin/data.json create mode 100644 tests/defaults/test_vault_suites/ignore_setting/ignore_setting.md create mode 100644 tests/defaults/test_vault_suites/ignore_setting/outside_of_scandir/not_supposed_to_be_scanned.md create mode 100644 tests/defaults/test_vault_suites/ignore_setting/scan_dir/ignored_by_setting_ignored/not_supposed_to_be_scanned.md create mode 100644 tests/defaults/test_vault_suites/ignore_setting/scan_dir/ignored_by_setting_ignored/some/other/subdir/not_supposed_to_be_scanned.md create mode 100644 tests/defaults/test_vault_suites/ignore_setting/scan_dir/included_file.md create mode 100644 tests/defaults/test_vault_suites/ignore_setting/scan_dir/some/other/subdir/also_included_file.md diff --git a/README.md b/README.md index 8ee902b0..18b04483 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,16 @@ Current features (check out the wiki for more details): * **Custom scan directory** * The plugin will scan the entire vault by default * You can also set which directory (includes all sub-directories as well) to scan via plugin settings +* **Ignore Folders and Files** + * You can specify which files and folders to ignore + * This can be done in the settings of this plugin with [Glob syntax](https://en.wikipedia.org/wiki/Glob_(programming)#Syntax). + * If you're working on your own globs, you can test them out [here](https://globster.xyz/) + * Examples: + * `**/*.excalidraw.md` - Ignore all files that end in `.excalidraw.md` + * => avoids excalidraw files from being scanned which can be extremely slow + * `Template/**` - Ignore all files in the `Template` folder (including subfolders) + * `**/private/**` - Ignore all files in folders that are called `private` no matter where they are in the vault + * `[Pp]rivate*/**` - Ignore all files and folders in the root of the vault that start with `private` or with `Private` * **Updating notes from file** - Your text files are the canonical source of the notes. * **Tags**, including **tags for an entire file**. * **Adding to user-specified deck** on a *per-file* basis. diff --git a/main.ts b/main.ts index 9a3fd742..4b4c1d30 100644 --- a/main.ts +++ b/main.ts @@ -1,7 +1,7 @@ import { Notice, Plugin, addIcon, TFile, TFolder } from 'obsidian' import * as AnkiConnect from './src/anki' import { PluginSettings, ParsedSettings } from './src/interfaces/settings-interface' -import { SettingsTab } from './src/settings' +import { DEFAULT_IGNORED_FILE_GLOBS, SettingsTab } from './src/settings' import { ANKI_ICON } from './src/constants' import { settingToData } from './src/setting-to-data' import { FileManager } from './src/files-manager' @@ -42,7 +42,8 @@ export default class MyPlugin extends Plugin { "CurlyCloze - Highlights to Clozes": false, "ID Comments": true, "Add Obsidian Tags": false, - } + }, + IGNORED_FILE_GLOBS: DEFAULT_IGNORED_FILE_GLOBS, } /*Making settings from scratch, so need note types*/ this.note_types = await AnkiConnect.invoke('modelNames') as Array diff --git a/package-lock.json b/package-lock.json index cfd65aca..a2372c62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "byte-base64": "^1.1.0", + "multimatch": "^7.0.0", "showdown": "^2.1.0", "ts-md5": "^1.2.7", "webdriver": "^8.5.5" @@ -1236,6 +1237,28 @@ "dequal": "^2.0.3" } }, + "node_modules/array-differ": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-4.0.0.tgz", + "integrity": "sha512-Q6VPTLMsmXZ47ENG3V+wQyZS1ZxXMxFyYzA+Z/GMrJ6yIutAIEf9wTyroTzmGjNfox9/h3GdGBCVh43GVFx4Uw==", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/array-union": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-3.0.1.tgz", + "integrity": "sha512-1OvF9IbWwaeiM9VhzYXVQacMibxpXOMYVNIvMtKRyX9SImBXpKcFr8XvFDeEslCyuH/t6KRt7HEO94AlP8Iatw==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/ast-types": { "version": "0.13.4", "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", @@ -4557,6 +4580,22 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/multimatch": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-7.0.0.tgz", + "integrity": "sha512-SYU3HBAdF4psHEL/+jXDKHO95/m5P2RvboHT2Y0WtTttvJLP4H/2WS9WlQPFvF6C8d6SpLw8vjCnQOnVIVOSJQ==", + "dependencies": { + "array-differ": "^4.0.0", + "array-union": "^3.0.1", + "minimatch": "^9.0.3" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/mute-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", diff --git a/package.json b/package.json index b45c88ef..6e15c554 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ }, "dependencies": { "byte-base64": "^1.1.0", + "multimatch": "^7.0.0", "showdown": "^2.1.0", "ts-md5": "^1.2.7", "webdriver": "^8.5.5" diff --git a/src/files-manager.ts b/src/files-manager.ts index 204dc8b4..fb4d35f3 100644 --- a/src/files-manager.ts +++ b/src/files-manager.ts @@ -4,7 +4,7 @@ import { App, TFile, TFolder, TAbstractFile, CachedMetadata, FileSystemAdapter, import { AllFile } from './file' import * as AnkiConnect from './anki' import { basename } from 'path' - +import multimatch from "multimatch" interface addNoteResponse { result: number, error: string | null @@ -63,16 +63,25 @@ export class FileManager { constructor(app: App, data:ParsedSettings, files: TFile[], file_hashes: Record, added_media: string[]) { this.app = app this.data = data - this.files = files + + this.files = this.findFilesThatAreNotIgnored(files, data); + this.ownFiles = [] this.file_hashes = file_hashes this.added_media_set = new Set(added_media) } - getUrl(file: TFile): string { return "obsidian://open?vault=" + encodeURIComponent(this.data.vault_name) + String.raw`&file=` + encodeURIComponent(file.path) } + findFilesThatAreNotIgnored(files:TFile[], data:ParsedSettings):TFile[]{ + let ignoredFiles = [] + ignoredFiles = multimatch(files.map(file => file.path), data.ignored_file_globs) + + let notIgnoredFiles = files.filter(file => !ignoredFiles.contains(file.path)) + return notIgnoredFiles; + } + getFolderPathList(file: TFile): TFolder[] { let result: TFolder[] = [] let abstractFile: TAbstractFile = file diff --git a/src/interfaces/settings-interface.ts b/src/interfaces/settings-interface.ts index 7190d6f5..dd022b0f 100644 --- a/src/interfaces/settings-interface.ts +++ b/src/interfaces/settings-interface.ts @@ -28,7 +28,8 @@ export interface PluginSettings { "CurlyCloze - Highlights to Clozes": boolean, "ID Comments": boolean, "Add Obsidian Tags": boolean - } + }, + IGNORED_FILE_GLOBS:string[] } export interface FileData { @@ -59,4 +60,5 @@ export interface ParsedSettings extends FileData { add_file_link: boolean folder_decks: Record folder_tags: Record + ignored_file_globs: string[] } diff --git a/src/setting-to-data.ts b/src/setting-to-data.ts index e30e0252..62c47fd2 100644 --- a/src/setting-to-data.ts +++ b/src/setting-to-data.ts @@ -42,6 +42,7 @@ export async function settingToData(app: App, settings: PluginSettings, fields_d result.comment = settings.Defaults["ID Comments"] result.add_context = settings.Defaults["Add Context"] result.add_obs_tags = settings.Defaults["Add Obsidian Tags"] + result.ignored_file_globs = settings.IGNORED_FILE_GLOBS ?? []; return result } diff --git a/src/settings.ts b/src/settings.ts index 54fae92d..08f42fe2 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -14,6 +14,10 @@ const defaultDescs = { "Add Obsidian Tags": "Interpret #tags in the fields of a note as Anki tags, removing them from the note text in Anki." } +export const DEFAULT_IGNORED_FILE_GLOBS = [ + '**/*.excalidraw.md' +]; + export class SettingsTab extends PluginSettingTab { setup_custom_regexp(note_type: string, row_cells: HTMLCollection) { @@ -397,6 +401,35 @@ export class SettingsTab extends PluginSettingTab { } ) } + setup_ignore_files() { + let { containerEl } = this; + const plugin = (this as any).plugin + let ignored_files_settings = containerEl.createEl('h3', { text: 'Ignored File Settings' }) + plugin.settings["IGNORED_FILE_GLOBS"] = plugin.settings.hasOwnProperty("IGNORED_FILE_GLOBS") ? plugin.settings["IGNORED_FILE_GLOBS"] : DEFAULT_IGNORED_FILE_GLOBS + const descriptionFragment = document.createDocumentFragment(); + descriptionFragment.createEl("span", { text: "Glob patterns for files to ignore. You can add multiple patterns. One per line. Have a look at the " }) + descriptionFragment.createEl("a", { text: "README.md", href: "https://github.com/Pseudonium/Obsidian_to_Anki?tab=readme-ov-file#features" }); + descriptionFragment.createEl("span", { text: " for more information, examples and further resources." }) + + + new Setting(ignored_files_settings) + .setName("Patterns to ignore") + .setDesc(descriptionFragment) + .addTextArea(text => { + text.setValue(plugin.settings.IGNORED_FILE_GLOBS.join("\n")) + .setPlaceholder("Examples: '**/*.excalidraw.md', 'Templates/**'") + .onChange((value) => { + let ignoreLines = value.split("\n") + ignoreLines = ignoreLines.filter(e => e.trim() != "") //filter out empty lines and blank lines + plugin.settings.IGNORED_FILE_GLOBS = ignoreLines + + plugin.saveAllData() + } + ) + text.inputEl.rows = 10 + text.inputEl.cols = 30 + }); + } setup_display() { let {containerEl} = this @@ -409,6 +442,7 @@ export class SettingsTab extends PluginSettingTab { this.setup_syntax() this.setup_defaults() this.setup_buttons() + this.setup_ignore_files() } async display() { diff --git a/tests/anki/test_ignore_setting.py b/tests/anki/test_ignore_setting.py new file mode 100644 index 00000000..ea6435c2 --- /dev/null +++ b/tests/anki/test_ignore_setting.py @@ -0,0 +1,82 @@ +import re +import pytest +from anki.errors import NotFoundError # noqa +from anki.collection import Collection +from anki.collection import SearchNode + +# from conftest import col + +test_name = "ignore_setting" +col_path = "tests/test_outputs/{}/Anki2/User 1/collection.anki2".format(test_name) + +test_file_paths = [ + "tests/test_outputs/{}/Obsidian/{}/scan_dir/included_file.md".format( + test_name, test_name + ), + "tests/test_outputs/{}/Obsidian/{}/scan_dir/some/other/subdir/also_included_file.md".format( + test_name, test_name + ), +] + +test_file_no_cards_paths = [ + "tests/test_outputs/{}/Obsidian/{}/outside_of_scandir/not_supposed_to_be_scanned.md".format( + test_name, test_name + ), + "tests/test_outputs/{}/Obsidian/{}/scan_dir/ignored_by_setting_ignored/not_supposed_to_be_scanned.md".format( + test_name, test_name + ), + "tests/test_outputs/{}/Obsidian/{}/scan_dir/ignored_by_setting_ignored/some/other/subdir/not_supposed_to_be_scanned.md".format( + test_name, test_name + ), +] + + +@pytest.fixture() +def col(): + col = Collection(col_path) + yield col + col.close() + + +def test_col_exists(col): + assert not col.is_empty() + + +def test_deck_default_exists(col: Collection): + assert col.decks.id_for_name("Default") is not None + + +def test_cards_count(col: Collection): + assert len(col.find_cards(col.build_search_string(SearchNode(deck="Default")))) == 6 + + +def test_cards_ids_from_obsidian(col: Collection): + ID_REGEXP_STR = r"\n?(?: +START +Basic +This is a test. Ouside of scandir. +Ouside of scandir is to be ignored +Back: Test successful! +Tags: Testing +END + + +START +Basic +Front: This is a test with Front specified. Outside of scandir. +Back: Test successful! +Tags: Testing 2 +END + + +START +Basic +This is a test. Outside of scandir. +And the test is continuing. +Back: Test successful! +END \ No newline at end of file diff --git a/tests/defaults/test_vault_suites/ignore_setting/scan_dir/ignored_by_setting_ignored/not_supposed_to_be_scanned.md b/tests/defaults/test_vault_suites/ignore_setting/scan_dir/ignored_by_setting_ignored/not_supposed_to_be_scanned.md new file mode 100644 index 00000000..0dd40fac --- /dev/null +++ b/tests/defaults/test_vault_suites/ignore_setting/scan_dir/ignored_by_setting_ignored/not_supposed_to_be_scanned.md @@ -0,0 +1,23 @@ + +START +Basic +This is a test. Parent dir is ignored. +Back: Test successful! +Tags: Testing +END + + +START +Basic +Front: This is a test with Front specified. Parent dir is ignored. +Back: Test successful! +Tags: Testing 2 +END + + +START +Basic +This is a test. Parent dir is ignored. +And the test is continuing. +Back: Test successful! +END \ No newline at end of file diff --git a/tests/defaults/test_vault_suites/ignore_setting/scan_dir/ignored_by_setting_ignored/some/other/subdir/not_supposed_to_be_scanned.md b/tests/defaults/test_vault_suites/ignore_setting/scan_dir/ignored_by_setting_ignored/some/other/subdir/not_supposed_to_be_scanned.md new file mode 100644 index 00000000..5574b2bc --- /dev/null +++ b/tests/defaults/test_vault_suites/ignore_setting/scan_dir/ignored_by_setting_ignored/some/other/subdir/not_supposed_to_be_scanned.md @@ -0,0 +1,23 @@ + +START +Basic +This is a test. Inside scandir, however parent dir is ignored. +Back: Test successful! +Tags: Testing +END + + +START +Basic +Front: This is a test with Front specified. Inside scandir, however parent dir is ignored. +Back: Test successful! +Tags: Testing 2 +END + + +START +Basic +This is a test. Inside scandir, however parent dir is ignored. +And the test is continuing. +Back: Test successful! +END \ No newline at end of file diff --git a/tests/defaults/test_vault_suites/ignore_setting/scan_dir/included_file.md b/tests/defaults/test_vault_suites/ignore_setting/scan_dir/included_file.md new file mode 100644 index 00000000..99d579ee --- /dev/null +++ b/tests/defaults/test_vault_suites/ignore_setting/scan_dir/included_file.md @@ -0,0 +1,24 @@ + + +START +Basic +This is a test. This file should be scanned for cards as it is in the scandir and not ignored. +Back: Test successful! +Tags: Testing +END + + +START +Basic +Front: This is a test with Front specified. This file should be scanned for cards as it is in the scandir and not ignored. +Back: Test successful! +Tags: Testing 2 +END + + +START +Basic +This is a test. This file should be scanned for cards as it is in the scandir and not ignored. +And the test is continuing. +Back: Test successful! +END diff --git a/tests/defaults/test_vault_suites/ignore_setting/scan_dir/some/other/subdir/also_included_file.md b/tests/defaults/test_vault_suites/ignore_setting/scan_dir/some/other/subdir/also_included_file.md new file mode 100644 index 00000000..5df9f45a --- /dev/null +++ b/tests/defaults/test_vault_suites/ignore_setting/scan_dir/some/other/subdir/also_included_file.md @@ -0,0 +1,24 @@ + + +START +Basic +This is a test. Subfolders should be scanned for cards. +Back: Test successful! +Tags: Testing +END + + +START +Basic +Front: This is a test with Front specified. Subfolders should be scanned for cards. +Back: Test successful! +Tags: Testing 2 +END + + +START +Basic +This is a test. Subfolders should be scanned for cards. +And the test is continuing. +Back: Test successful! +END