From a9b3cf6068993c3ce9fa76f25b7b7c5c253c0ab3 Mon Sep 17 00:00:00 2001 From: war-in Date: Tue, 16 Apr 2024 13:17:02 +0200 Subject: [PATCH 1/8] to markdown & to text --- lib/ExpensiMark.d.ts | 6 ++++-- lib/ExpensiMark.js | 41 ++++++++++++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/lib/ExpensiMark.d.ts b/lib/ExpensiMark.d.ts index 280f3871..c21cea13 100644 --- a/lib/ExpensiMark.d.ts +++ b/lib/ExpensiMark.d.ts @@ -89,14 +89,16 @@ export default class ExpensiMark { * Replaces HTML with markdown * * @param htmlString + * @param extras */ - htmlToMarkdown(htmlString: string): string; + htmlToMarkdown(htmlString: string, extras?: Object): string; /** * Convert HTML to text * * @param htmlString + * @param extras */ - htmlToText(htmlString: string): string; + htmlToText(htmlString: string, extras?: Object): string; /** * Modify text for Quotes replacing chevrons with html elements * diff --git a/lib/ExpensiMark.js b/lib/ExpensiMark.js index ae306472..00005136 100644 --- a/lib/ExpensiMark.js +++ b/lib/ExpensiMark.js @@ -124,7 +124,7 @@ export default class ExpensiMark { name: 'image', regex: MARKDOWN_IMAGE_REGEX, replacement: (match, g1, g2) => `${this.escapeAttributeContent(g1)}`, - rawInputReplacement: (match, g1, g2) => `${this.escapeAttributeContent(g1)}` + rawInputReplacement: (match, g1, g2) => `${this.escapeAttributeContent(g1)}` }, /** @@ -264,7 +264,11 @@ export default class ExpensiMark { this.currentQuoteDepth++; } - const replacedText = this.replace(textToReplace, {filterRules, shouldEscapeText: false, shouldKeepRawInput}); + const replacedText = this.replace(textToReplace, { + filterRules, + shouldEscapeText: false, + shouldKeepRawInput + }); this.currentQuoteDepth = 0; return `
${isStartingWithSpace ? ' ' : ''}${replacedText}
`; }, @@ -453,6 +457,16 @@ export default class ExpensiMark { return `!(${g2})`; } + }, + { + name: 'roomMention', + regex: //gi, + replacement: (match, g1, extras) => extras.reportIdToName[g1] + }, + { + name: 'userMention', + regex: //gi, + replacement: (match, g1, extras) => extras.accountIdToName[g1] } ]; @@ -497,11 +511,21 @@ export default class ExpensiMark { regex: /<]*src\s*=\s*(['"])(.*?)\1(?:[^><]*alt\s*=\s*(['"])(.*?)\3)?[^><]*>*(?![^<][\s\S]*?(<\/pre>|<\/code>))/gi, replacement: '[Attachment]', }, + { + name: 'roomMention', + regex: //gi, + replacement: (match, g1, extras) => extras.reportIdToName[g1] + }, + { + name: 'userMention', + regex: //gi, + replacement: (match, g1, extras) => extras.accountIdToName[g1] + }, { name: 'stripTag', regex: /(<([^>]+)>)/gi, replacement: '', - }, + } ]; /** @@ -768,10 +792,11 @@ export default class ExpensiMark { * Replaces HTML with markdown * * @param {String} htmlString + * @param {Object} extras * * @returns {String} */ - htmlToMarkdown(htmlString) { + htmlToMarkdown(htmlString, extras = {}) { let generatedMarkdown = htmlString; const body = /<(body)(?:"[^"]*"|'[^']*'|[^'"><])*>(?:\n|\r\n)?([\s\S]*?)(?:\n|\r\n)?<\/\1>(?![^<]*(<\/pre>|<\/code>))/im; const parseBodyTag = generatedMarkdown.match(body); @@ -786,7 +811,7 @@ export default class ExpensiMark { if (rule.pre) { generatedMarkdown = rule.pre(generatedMarkdown); } - generatedMarkdown = generatedMarkdown.replace(rule.regex, rule.replacement); + generatedMarkdown = generatedMarkdown.replace(rule.regex, typeof rule.replacement === "function" && extras ? (match, g1) => rule.replacement(match, g1, extras) : rule.replacement); }); return Str.htmlDecode(this.replaceBlockElementWithNewLine(generatedMarkdown)); } @@ -795,13 +820,15 @@ export default class ExpensiMark { * Convert HTML to text * * @param {String} htmlString + * @param {Object} extras + * * @returns {String} */ - htmlToText(htmlString) { + htmlToText(htmlString, extras) { let replacedText = htmlString; this.htmlToTextRules.forEach((rule) => { - replacedText = replacedText.replace(rule.regex, rule.replacement); + replacedText = replacedText.replace(rule.regex, typeof rule.replacement === "function" && extras ? (match, g1) => rule.replacement(match, g1, extras) : rule.replacement); }); // Unescaping because the text is escaped in 'replace' function From f81aca3c7d37118aa3ed3b1ecbcae99ca99bd8eb Mon Sep 17 00:00:00 2001 From: war-in Date: Tue, 16 Apr 2024 13:17:07 +0200 Subject: [PATCH 2/8] add tests --- __tests__/ExpensiMark-HTMLToText-test.js | 32 +++++++++++++++++++++++- __tests__/ExpensiMark-Markdown-test.js | 32 +++++++++++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/__tests__/ExpensiMark-HTMLToText-test.js b/__tests__/ExpensiMark-HTMLToText-test.js index 7a65d877..197010d8 100644 --- a/__tests__/ExpensiMark-HTMLToText-test.js +++ b/__tests__/ExpensiMark-HTMLToText-test.js @@ -133,7 +133,7 @@ test('Test remove style tag', () => { expect(parser.htmlToText(testString)).toBe('a text'); }); -test('Mention html to text', () => { +test('Mention user html to text', () => { let testString = '@user@domain.com'; expect(parser.htmlToText(testString)).toBe('@user@domain.com'); @@ -145,6 +145,36 @@ test('Mention html to text', () => { testString = '@user@DOMAIN.com'; expect(parser.htmlToText(testString)).toBe('@user@DOMAIN.com'); + + const extras = { + "accountIdToName": { + "1234": "@user@domain.com" + } + } + testString = ''; + expect(parser.htmlToText(testString, extras)).toBe('@user@domain.com'); +}); + +test('Mention report html to text', () => { + let testString = '#room-name'; + expect(parser.htmlToText(testString)).toBe('#room-name'); + + testString = '#ROOM-NAME'; + expect(parser.htmlToText(testString)).toBe('#ROOM-NAME'); + + testString = '#ROOM-name'; + expect(parser.htmlToText(testString)).toBe('#ROOM-name'); + + testString = '#room-NAME'; + expect(parser.htmlToText(testString)).toBe('#room-NAME'); + + const extras = { + "reportIdToName": { + "1234": "#room-name" + } + } + testString = ''; + expect(parser.htmlToText(testString, extras)).toBe('#room-name'); }); test('Test replacement for tags', () => { diff --git a/__tests__/ExpensiMark-Markdown-test.js b/__tests__/ExpensiMark-Markdown-test.js index cd68a4b8..2f2b5a0b 100644 --- a/__tests__/ExpensiMark-Markdown-test.js +++ b/__tests__/ExpensiMark-Markdown-test.js @@ -743,7 +743,7 @@ test('Linebreak should be remained for text between code block', () => { }); }); -test('Mention html to markdown', () => { +test('Mention user html to markdown', () => { let testString = '@user@domain.com'; expect(parser.htmlToMarkdown(testString)).toBe('@user@domain.com'); @@ -755,6 +755,36 @@ test('Mention html to markdown', () => { testString = '@user@DOMAIN.com'; expect(parser.htmlToMarkdown(testString)).toBe('@user@DOMAIN.com'); + + const extras = { + "accountIdToName": { + "1234": "@user@domain.com" + } + } + testString = ''; + expect(parser.htmlToMarkdown(testString, extras)).toBe('@user@domain.com'); +}); + +test('Mention report html to markdown', () => { + let testString = '#room-name'; + expect(parser.htmlToMarkdown(testString)).toBe('#room-name'); + + testString = '#ROOM-NAME'; + expect(parser.htmlToMarkdown(testString)).toBe('#ROOM-NAME'); + + testString = '#ROOM-name'; + expect(parser.htmlToMarkdown(testString)).toBe('#ROOM-name'); + + testString = '#room-NAME'; + expect(parser.htmlToMarkdown(testString)).toBe('#room-NAME'); + + const extras = { + "reportIdToName": { + "1234": "#room-name" + } + } + testString = ''; + expect(parser.htmlToMarkdown(testString, extras)).toBe('#room-name'); }); describe('Image tag conversion to markdown', () => { From 9848b00d33274569d835cdc3f19cdaa523bff14f Mon Sep 17 00:00:00 2001 From: Robert Kozik Date: Wed, 17 Apr 2024 20:45:37 +0200 Subject: [PATCH 3/8] update ExpensiMark types to include extras object --- lib/ExpensiMark.d.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/ExpensiMark.d.ts b/lib/ExpensiMark.d.ts index c21cea13..9bb1923b 100644 --- a/lib/ExpensiMark.d.ts +++ b/lib/ExpensiMark.d.ts @@ -1,10 +1,11 @@ -declare type Replacement = (...args: string[]) => string; +declare type Replacement = (...args: string[], extras?: ExtrasObject) => string; declare type Name = | 'codeFence' | 'inlineCodeBlock' | 'email' | 'link' | 'hereMentions' + | 'roomMentions' | 'userMentions' | 'autoEmail' | 'autolink' @@ -32,6 +33,11 @@ declare type Rule = { pre?: (input: string) => string; post?: (input: string) => string; }; + +declare type ExtrasObject = { + reportIdToName?: Record; + accountIDToName?: Record; +}; export default class ExpensiMark { rules: Rule[]; htmlToMarkdownRules: Rule[]; @@ -91,14 +97,14 @@ export default class ExpensiMark { * @param htmlString * @param extras */ - htmlToMarkdown(htmlString: string, extras?: Object): string; + htmlToMarkdown(htmlString: string, extras?: ExtrasObject): string; /** * Convert HTML to text * * @param htmlString * @param extras */ - htmlToText(htmlString: string, extras?: Object): string; + htmlToText(htmlString: string, extras?: ExtrasObject): string; /** * Modify text for Quotes replacing chevrons with html elements * From d8a754d8597b110d7a0fbc981335131ecf678a99 Mon Sep 17 00:00:00 2001 From: Robert Kozik Date: Thu, 18 Apr 2024 21:09:52 +0200 Subject: [PATCH 4/8] update tests HTML->Markdown & HTML->Text for mentions --- __tests__/ExpensiMark-HTMLToText-test.js | 22 ++++++++++++++-------- __tests__/ExpensiMark-Markdown-test.js | 22 ++++++++++++++-------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/__tests__/ExpensiMark-HTMLToText-test.js b/__tests__/ExpensiMark-HTMLToText-test.js index 197010d8..43a09a10 100644 --- a/__tests__/ExpensiMark-HTMLToText-test.js +++ b/__tests__/ExpensiMark-HTMLToText-test.js @@ -147,10 +147,13 @@ test('Mention user html to text', () => { expect(parser.htmlToText(testString)).toBe('@user@DOMAIN.com'); const extras = { - "accountIdToName": { - "1234": "@user@domain.com" - } - } + accountIdToName: { + '1234': 'user@domain.com', + }, + }; + testString = ''; + expect(parser.htmlToText(testString, extras)).toBe('@user@domain.com'); + testString = ''; expect(parser.htmlToText(testString, extras)).toBe('@user@domain.com'); }); @@ -169,10 +172,13 @@ test('Mention report html to text', () => { expect(parser.htmlToText(testString)).toBe('#room-NAME'); const extras = { - "reportIdToName": { - "1234": "#room-name" - } - } + reportIdToName: { + '1234': '#room-name', + }, + }; + testString = ''; + expect(parser.htmlToText(testString, extras)).toBe('#room-name'); + testString = ''; expect(parser.htmlToText(testString, extras)).toBe('#room-name'); }); diff --git a/__tests__/ExpensiMark-Markdown-test.js b/__tests__/ExpensiMark-Markdown-test.js index 2f2b5a0b..49ea5af2 100644 --- a/__tests__/ExpensiMark-Markdown-test.js +++ b/__tests__/ExpensiMark-Markdown-test.js @@ -757,10 +757,13 @@ test('Mention user html to markdown', () => { expect(parser.htmlToMarkdown(testString)).toBe('@user@DOMAIN.com'); const extras = { - "accountIdToName": { - "1234": "@user@domain.com" - } - } + accountIdToName: { + '1234': 'user@domain.com', + }, + }; + testString = ''; + expect(parser.htmlToMarkdown(testString, extras)).toBe('@user@domain.com'); + testString = ''; expect(parser.htmlToMarkdown(testString, extras)).toBe('@user@domain.com'); }); @@ -779,10 +782,13 @@ test('Mention report html to markdown', () => { expect(parser.htmlToMarkdown(testString)).toBe('#room-NAME'); const extras = { - "reportIdToName": { - "1234": "#room-name" - } - } + reportIdToName: { + '1234': '#room-name', + }, + }; + testString = ''; + expect(parser.htmlToMarkdown(testString, extras)).toBe('#room-name'); + testString = ''; expect(parser.htmlToMarkdown(testString, extras)).toBe('#room-name'); }); From 82c6f2144dff81c69aceca2a044bf2b1aac691a8 Mon Sep 17 00:00:00 2001 From: Robert Kozik Date: Wed, 17 Apr 2024 20:54:21 +0200 Subject: [PATCH 5/8] handle extras object as lats argument of replacement function --- lib/ExpensiMark.js | 54 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/lib/ExpensiMark.js b/lib/ExpensiMark.js index 00005136..d1e7d8af 100644 --- a/lib/ExpensiMark.js +++ b/lib/ExpensiMark.js @@ -460,14 +460,28 @@ export default class ExpensiMark { }, { name: 'roomMention', - regex: //gi, - replacement: (match, g1, extras) => extras.reportIdToName[g1] + regex: //gi, + replacement: (match, g1, offset, string, extras) => { + const reportToNameMap = extras.reportIdToName; + if (!reportToNameMap || !reportToNameMap[g1]) { + return ''; + } + + return reportToNameMap[g1]; + }, }, { name: 'userMention', - regex: //gi, - replacement: (match, g1, extras) => extras.accountIdToName[g1] - } + regex: //gi, + replacement: (match, g1, offset, string, extras) => { + const accountToNameMap = extras.accountIdToName; + if (!accountToNameMap || !accountToNameMap[g1]) { + return ''; + } + + return `@${extras.accountIdToName[g1]}`; + }, + }, ]; /** @@ -513,13 +527,27 @@ export default class ExpensiMark { }, { name: 'roomMention', - regex: //gi, - replacement: (match, g1, extras) => extras.reportIdToName[g1] + regex: //gi, + replacement: (match, g1, offset, string, extras) => { + const reportToNameMap = extras.reportIdToName; + if (!reportToNameMap || !reportToNameMap[g1]) { + return ''; + } + + return reportToNameMap[g1]; + }, }, { name: 'userMention', - regex: //gi, - replacement: (match, g1, extras) => extras.accountIdToName[g1] + regex: //gi, + replacement: (match, g1, offset, string, extras) => { + const accountToNameMap = extras.accountIdToName; + if (!accountToNameMap || !accountToNameMap[g1]) { + return ''; + } + + return `@${extras.accountIdToName[g1]}`; + }, }, { name: 'stripTag', @@ -811,7 +839,10 @@ export default class ExpensiMark { if (rule.pre) { generatedMarkdown = rule.pre(generatedMarkdown); } - generatedMarkdown = generatedMarkdown.replace(rule.regex, typeof rule.replacement === "function" && extras ? (match, g1) => rule.replacement(match, g1, extras) : rule.replacement); + + // if replacement is a function, we want to pass optional extras to it + const replacementFunction = typeof rule.replacement === 'function' ? (...args) => rule.replacement(...args, extras) : rule.replacement; + generatedMarkdown = generatedMarkdown.replace(rule.regex, replacementFunction); }); return Str.htmlDecode(this.replaceBlockElementWithNewLine(generatedMarkdown)); } @@ -828,7 +859,8 @@ export default class ExpensiMark { let replacedText = htmlString; this.htmlToTextRules.forEach((rule) => { - replacedText = replacedText.replace(rule.regex, typeof rule.replacement === "function" && extras ? (match, g1) => rule.replacement(match, g1, extras) : rule.replacement); + const replacementFunction = typeof rule.replacement === 'function' ? (...args) => rule.replacement(...args, extras) : rule.replacement; + replacedText = replacedText.replace(rule.regex, replacementFunction); }); // Unescaping because the text is escaped in 'replace' function From 2540b8ec9936503967bd2bd3601bd03b742618c1 Mon Sep 17 00:00:00 2001 From: Robert Kozik Date: Thu, 18 Apr 2024 21:07:28 +0200 Subject: [PATCH 6/8] add disabledRules for replace function --- __tests__/ExpensiMark-HTML-test.js | 7 +++++++ lib/ExpensiMark.d.ts | 8 ++++++-- lib/ExpensiMark.js | 24 ++++++++++++++++++++---- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/__tests__/ExpensiMark-HTML-test.js b/__tests__/ExpensiMark-HTML-test.js index eb72e625..d1c5e0b5 100644 --- a/__tests__/ExpensiMark-HTML-test.js +++ b/__tests__/ExpensiMark-HTML-test.js @@ -2030,6 +2030,13 @@ describe('room mentions', () => { expect(parser.replace(testString)).toBe(resultString); }); + test('room mention shouldn\'t be parsed when rule is disabled', () => { + const testString = '*hello* @user@mail.com in #room!'; + const resultString = 'hello @user@mail.com in #room!'; + const disabledRules = ['reportMentions']; + expect(parser.replace(testString, {disabledRules})).toBe(resultString); + }); + test('room mention with italic, bold and strikethrough styles', () => { const testString = '#room' + ' _#room_' diff --git a/lib/ExpensiMark.d.ts b/lib/ExpensiMark.d.ts index 9bb1923b..291c8492 100644 --- a/lib/ExpensiMark.d.ts +++ b/lib/ExpensiMark.d.ts @@ -7,6 +7,7 @@ declare type Name = | 'hereMentions' | 'roomMentions' | 'userMentions' + | 'reportMentions' | 'autoEmail' | 'autolink' | 'quote' @@ -49,7 +50,9 @@ export default class ExpensiMark { * @param text - Text to parse as markdown * @param options - Options to customize the markdown parser * @param options.filterRules=[] - An array of name of rules as defined in this class. - * If not provided, all available rules will be applied. + * If not provided, all available rules will be applied. If provided, only the rules in the array will be applied. + * @param options.disabledRules=[] - An array of name of rules as defined in this class. + * If not provided, all available rules will be applied. If provided, the rules in the array will be skipped. * @param options.shouldEscapeText=true - Whether or not the text should be escaped * @param options.shouldKeepRawInput=false - Whether or not the raw input should be kept and returned */ @@ -60,7 +63,8 @@ export default class ExpensiMark { shouldEscapeText, shouldKeepRawInput, }?: { - filterRules?: string[]; + filterRules?: Name[]; + disabledRules?: Name[]; shouldEscapeText?: boolean; shouldKeepRawInput?: boolean; }, diff --git a/lib/ExpensiMark.js b/lib/ExpensiMark.js index d1e7d8af..1dcc85f8 100644 --- a/lib/ExpensiMark.js +++ b/lib/ExpensiMark.js @@ -174,7 +174,7 @@ export default class ExpensiMark { * combination of letters and hyphens */ { - name: 'roomMentions', + name: 'reportMentions', regex: /(?$1', @@ -581,6 +581,20 @@ export default class ExpensiMark { this.currentQuoteDepth = 0; } + getHtmlRuleset(filterRules, disabledRules, shouldKeepRawInput) { + let rules = this.rules; + if(shouldKeepRawInput) { + rules = this.shouldKeepWhitespaceRules; + } + if (!_.isEmpty(filterRules)) { + rules = _.filter(this.rules, (rule) => _.contains(filterRules, rule.name)); + } + if (!_.isEmpty(disabledRules)) { + rules = _.filter(rules, (rule) => !_.contains(disabledRules, rule.name)); + } + return rules; + } + /** * Replaces markdown with html elements * @@ -589,14 +603,16 @@ export default class ExpensiMark { * @param {String[]} [options.filterRules=[]] - An array of name of rules as defined in this class. * If not provided, all available rules will be applied. * @param {Boolean} [options.shouldEscapeText=true] - Whether or not the text should be escaped + * @param {String[]} [options.disabledRules=[]] - An array of name of rules as defined in this class. + * If not provided, all available rules will be applied. If provided, the rules in the array will be skipped. * * @returns {String} */ - replace(text, {filterRules = [], shouldEscapeText = true, shouldKeepRawInput = false} = {}) { + replace(text, {filterRules = [], shouldEscapeText = true, shouldKeepRawInput = false, disabledRules = []} = {}) { // This ensures that any html the user puts into the comment field shows as raw html let replacedText = shouldEscapeText ? _.escape(text) : text; - const enabledRules = shouldKeepRawInput ? this.shouldKeepWhitespaceRules : this.rules; - const rules = _.isEmpty(filterRules) ? enabledRules : _.filter(this.rules, (rule) => _.contains(filterRules, rule.name)); + const rules = this.getHtmlRuleset(filterRules, disabledRules, shouldKeepRawInput); + try { rules.forEach((rule) => { // Pre-process text before applying regex From 7596ae07c9cd5a6b5265897034898b4526d681dc Mon Sep 17 00:00:00 2001 From: Robert Kozik Date: Fri, 19 Apr 2024 11:02:05 +0200 Subject: [PATCH 7/8] review changes --- lib/ExpensiMark.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/ExpensiMark.js b/lib/ExpensiMark.js index 1dcc85f8..5c6a5933 100644 --- a/lib/ExpensiMark.js +++ b/lib/ExpensiMark.js @@ -459,7 +459,7 @@ export default class ExpensiMark { } }, { - name: 'roomMention', + name: 'reportMentions', regex: //gi, replacement: (match, g1, offset, string, extras) => { const reportToNameMap = extras.reportIdToName; @@ -526,7 +526,7 @@ export default class ExpensiMark { replacement: '[Attachment]', }, { - name: 'roomMention', + name: 'reportMentions', regex: //gi, replacement: (match, g1, offset, string, extras) => { const reportToNameMap = extras.reportIdToName; @@ -875,6 +875,8 @@ export default class ExpensiMark { let replacedText = htmlString; this.htmlToTextRules.forEach((rule) => { + + // if replacement is a function, we want to pass optional extras to it const replacementFunction = typeof rule.replacement === 'function' ? (...args) => rule.replacement(...args, extras) : rule.replacement; replacedText = replacedText.replace(rule.regex, replacementFunction); }); From f41aae68380cf6dd3ac5ad25d8129fc8bf1591d4 Mon Sep 17 00:00:00 2001 From: Robert Kozik Date: Fri, 19 Apr 2024 20:44:48 +0200 Subject: [PATCH 8/8] remove wrong name type --- lib/ExpensiMark.d.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ExpensiMark.d.ts b/lib/ExpensiMark.d.ts index 291c8492..f431df7e 100644 --- a/lib/ExpensiMark.d.ts +++ b/lib/ExpensiMark.d.ts @@ -5,7 +5,6 @@ declare type Name = | 'email' | 'link' | 'hereMentions' - | 'roomMentions' | 'userMentions' | 'reportMentions' | 'autoEmail'