diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/docs/src/content/docs/reference/scripts/md.md b/docs/src/content/docs/reference/scripts/md.md new file mode 100644 index 0000000000..0ee11e1396 --- /dev/null +++ b/docs/src/content/docs/reference/scripts/md.md @@ -0,0 +1,34 @@ +--- +title: Markdown +sidebar: + order: 9.2 +keywords: markdown, mdx, frontmatter +--- + +The `MD` class contains several helpers to work with [Markdown](https://www.markdownguide.org/cheat-sheet/) and [frontmatter text](https://jekyllrb.com/docs/front-matter/). + +The parser also support markdown variants like [MDX](https://mdxjs.com/). + +## `frontmatter` + +Extracts and parses the frontmatter text from a markdown file. Returns `undefined` if no frontmatter is not found or the parsing failed. Default format is `yaml`. + +```javascript +const frontmatter = MD.frontmatter(text, "yaml") +``` + +## `content` + +Extracts the markdown source without the frontmatter. + +```javascript +const content = MD.content(text) +``` + +## `updateFrontmatter` + +Merges frontmatter values into the existing in a markdown file. Use `null` value to delete fields. + +```javascript +const updated = MD.updateFrontmatter(text, { title: "New Title" }) +``` diff --git a/docs/src/content/docs/reference/scripts/parsers.md b/docs/src/content/docs/reference/scripts/parsers.md index 9f3228238f..6ea0179849 100644 --- a/docs/src/content/docs/reference/scripts/parsers.md +++ b/docs/src/content/docs/reference/scripts/parsers.md @@ -129,7 +129,7 @@ title: "Hello, World!" ... ``` -You can use the `parsers.frontmatter` to parse out the metadata into an object +You can use the `parsers.frontmatter` or [MD](./md.md) to parse out the metadata into an object ```js const meta = parsers.frontmatter(file) diff --git a/genaisrc/blog-generator.genai.mjs b/genaisrc/blog-generator.genai.mjs index 693142c6aa..cbd7172d21 100644 --- a/genaisrc/blog-generator.genai.mjs +++ b/genaisrc/blog-generator.genai.mjs @@ -170,8 +170,15 @@ defOutputProcessor((output) => { if (/\`\`\`markdown\n/.test(md)) { md = md.replace(/\`\`\`markdown\n/g, "").replace(/\`\`\`\n?$/g, "") } - const fm = parsers.frontmatter(md) + const fm = MD.frontmatter(md) if (!fm) throw new Error("No frontmatter found") + + md = MD.updateFrontmatter(md, { + draft: true, + date: formattedDate, + authors: "genaiscript", + }) + const fn = `docs/src/content/docs/blog/drafts/${fm.title.replace(/[^a-z0-9]+/gi, "-")}.md`.toLocaleLowerCase() const sn = diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/core/src/annotations.test.ts b/packages/core/src/annotations.test.ts index 784f0cd2ad..8e600dbbec 100644 --- a/packages/core/src/annotations.test.ts +++ b/packages/core/src/annotations.test.ts @@ -13,7 +13,7 @@ describe("annotations", () => { ` const diags = parseAnnotations(output) - console.log(diags) + // console.log(diags) assert.strictEqual(diags.length, 3) assert.strictEqual(diags[0].severity, "error") assert.strictEqual(diags[0].filename, "packages/core/src/github.ts") diff --git a/packages/core/src/frontmatter.test.ts b/packages/core/src/frontmatter.test.ts index d1044d45b9..cda6dffa3a 100644 --- a/packages/core/src/frontmatter.test.ts +++ b/packages/core/src/frontmatter.test.ts @@ -1,6 +1,11 @@ import { describe, test } from "node:test" import assert from "node:assert/strict" -import { frontmatterTryParse } from "./frontmatter" +import { + frontmatterTryParse, + splitMarkdown, + updateFrontmatter, +} from "./frontmatter" +import { YAMLTryParse } from "./yaml" describe("replace frontmatter", () => { test("only", () => { @@ -21,3 +26,59 @@ foo bar assert.deepEqual(res, { foo: "bar" }) }) }) + +describe("splitMarkdown", () => { + test("split markdown with yaml frontmatter", () => { + const markdown = `--- +title: Test +--- +This is a test.` + const { frontmatter, content } = splitMarkdown(markdown) + assert.deepEqual(YAMLTryParse(frontmatter), { title: "Test" }) + assert.equal(content, "This is a test.") + }) + + test("split markdown with json frontmatter", () => { + const markdown = `--- +{ + "title": "Test" +} +--- +This is a test.` + const { frontmatter, content } = splitMarkdown(markdown) + assert.deepEqual(JSON.parse(frontmatter), { title: "Test" }) + assert.equal(content, "This is a test.") + }) +}) + +describe("updateFrontmatter", () => { + test("update yaml frontmatter", () => { + const markdown = `--- +title: Old Title +foo: bar +--- +This is a test.` + const newFrontmatter: any = { title: "New Title", foo: null } + const updatedMarkdown = updateFrontmatter(markdown, newFrontmatter) + const { frontmatter, content } = splitMarkdown(updatedMarkdown) + assert.deepEqual(YAMLTryParse(frontmatter), { title: "New Title" }) + assert.equal(content, "This is a test.") + }) + + test("update json frontmatter", () => { + const markdown = `--- +{ + "title": "Old Title", +"foo": "bar" +} +--- +This is a test.` + const newFrontmatter: any = { title: "New Title", foo: null } + const updatedMarkdown = updateFrontmatter(markdown, newFrontmatter, { + format: "json", + }) + const { frontmatter, content } = splitMarkdown(updatedMarkdown) + assert.deepEqual(JSON.parse(frontmatter), { title: "New Title" }) + assert.equal(content, "This is a test.") + }) +}) diff --git a/packages/core/src/frontmatter.ts b/packages/core/src/frontmatter.ts index 656bec8631..a6a661bbe8 100644 --- a/packages/core/src/frontmatter.ts +++ b/packages/core/src/frontmatter.ts @@ -1,36 +1,84 @@ +import { objectFromMap } from "pdfjs-dist/types/src/shared/util" import { JSON5TryParse } from "./json5" import { TOMLTryParse } from "./toml" -import { YAMLTryParse } from "./yaml" +import { YAMLTryParse, YAMLStringify } from "./yaml" export function frontmatterTryParse( text: string, - options?: { format: "yaml" | "json" | "toml" } -): { end: number; value: any } { - if (!text) return undefined - + options?: { format: "yaml" | "json" | "toml" | "text" } +): { text: string; value: any; endLine?: number } | undefined { const { format = "yaml" } = options || {} + const { frontmatter, endLine } = splitMarkdown(text) + if (!frontmatter) return undefined + + let res: any + switch (format) { + case "text": + res = frontmatter + break + case "json": + res = JSON5TryParse(frontmatter) + break + case "toml": + res = TOMLTryParse(frontmatter) + break + default: + res = YAMLTryParse(frontmatter) + break + } + return { text: frontmatter, value: res, endLine } +} +export function splitMarkdown(text: string): { + frontmatter?: string + endLine?: number + content: string +} { + if (!text) return { content: text } const lines = text.split(/\r?\n/g) const delimiter = "---" - if (lines[0] !== delimiter) return undefined + if (lines[0] !== delimiter) return { content: text } let end = 1 while (end < lines.length) { if (lines[end] === delimiter) break end++ } - if (end >= lines.length) return undefined - const fm = lines.slice(1, end).join("\n") - let res: any + if (end >= lines.length) return { content: text } + const frontmatter = lines.slice(1, end).join("\n") + const content = lines.slice(end + 1).join("\n") + return { frontmatter, content, endLine: end } +} + +export function updateFrontmatter( + text: string, + newFrontmatter: any, + options?: { format: "yaml" | "json" } +): string { + const { content = "" } = splitMarkdown(text) + if (newFrontmatter === null) return content + + const frontmatter = frontmatterTryParse(text, options)?.value ?? {} + + // merge object + for (const [key, value] of Object.entries(newFrontmatter ?? {})) { + if (value === null) { + delete frontmatter[key] + } else { + frontmatter[key] = value + } + } + + const { format = "yaml" } = options || {} + let fm: string switch (format) { case "json": - res = JSON5TryParse(fm) + fm = JSON.stringify(frontmatter, null, 2) break - case "toml": - res = TOMLTryParse(fm) + case "yaml": + fm = YAMLStringify(frontmatter) break default: - res = YAMLTryParse(fm) - break + throw new Error(`Unsupported format: ${format}`) } - return res !== undefined ? { end: end + 1, value: res } : undefined + return `---\n${fm}\n---\n${content}` } diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/core/src/importprompt.ts b/packages/core/src/importprompt.ts index efddb73873..8c4538bf7d 100644 --- a/packages/core/src/importprompt.ts +++ b/packages/core/src/importprompt.ts @@ -34,6 +34,7 @@ export async function importPrompt( "INI", "CSV", "XML", + "MD", "JSONL", "AICI", "fetchText", diff --git a/packages/core/src/promptcontext.ts b/packages/core/src/promptcontext.ts index ab6acee5e8..4cc0a58f36 100644 --- a/packages/core/src/promptcontext.ts +++ b/packages/core/src/promptcontext.ts @@ -57,6 +57,11 @@ import { resolveModelConnectionInfo } from "./models" import { resolveLanguageModel } from "./lm" import { callExpander } from "./expander" import { Project } from "./ast" +import { + frontmatterTryParse, + splitMarkdown, + updateFrontmatter, +} from "./frontmatter" export async function createPromptContext( prj: Project, @@ -84,6 +89,13 @@ export async function createPromptContext( const XML = Object.freeze({ parse: XMLParse, }) + const MD = Object.freeze({ + frontmatter: (text, format) => + frontmatterTryParse(text, { format })?.value ?? {}, + content: (text) => splitMarkdown(text)?.content, + updateFrontmatter: (text, frontmatter, format): string => + updateFrontmatter(text, frontmatter, { format }), + }) const JSONL = Object.freeze({ parse: JSONLTryParse, stringify: JSONLStringify, @@ -247,6 +259,7 @@ export async function createPromptContext( INI, AICI, XML, + MD, JSONL, retrieval, host: promptHost, diff --git a/packages/core/src/toml.ts b/packages/core/src/toml.ts index 30884a10d5..2425b4e4f9 100644 --- a/packages/core/src/toml.ts +++ b/packages/core/src/toml.ts @@ -8,3 +8,4 @@ export function TOMLTryParse(text: string, options?: { defaultValue?: any }) { return options?.defaultValue } } + diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index c92087f3c8..93c9e20a9b 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -1125,6 +1125,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1687,5 +1712,6 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } diff --git a/packages/core/src/types/prompt_type.d.ts b/packages/core/src/types/prompt_type.d.ts index 4f69c5f95d..12705da3ef 100644 --- a/packages/core/src/types/prompt_type.d.ts +++ b/packages/core/src/types/prompt_type.d.ts @@ -131,6 +131,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */ diff --git a/packages/vscode/src/docsnotebook.ts b/packages/vscode/src/docsnotebook.ts index a046a6c35b..c9774238af 100644 --- a/packages/vscode/src/docsnotebook.ts +++ b/packages/vscode/src/docsnotebook.ts @@ -385,7 +385,7 @@ function parseMarkdown(content: string): RawNotebookCell[] { // eat frontmatter const frontmatter = frontmatterTryParse(content) if (frontmatter) { - i = frontmatter.end + i = frontmatter.endLine cells.push({ language: "yaml", content: YAMLStringify(frontmatter.value), diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 6fd07baf48..408713069c 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -1151,6 +1151,31 @@ interface XML { parse(text: string, options?: XMLParseOptions): any } +interface MD { + /** + * Parses front matter from markdown + * @param text + */ + frontmatter(text: string, format?: "yaml" | "json" | "toml" | "text"): any + + /** + * Removes the front matter from the markdown text + */ + content(text: string): string + + /** + * Merges frontmatter with the existing text + * @param text + * @param frontmatter + * @param format + */ + updateFrontmatter( + text: string, + frontmatter: any, + format?: "yaml" | "json" + ): string +} + interface JSONL { /** * Parses a JSONL string to an array of objects @@ -1713,6 +1738,7 @@ interface PromptContext extends ChatGenerationContext { CSV: CSV INI: INI AICI: AICI + MD: MD host: PromptHost } @@ -1849,6 +1875,11 @@ declare var CSV: CSV */ declare var XML: XML +/** + * Markdown and frontmatter parsing. + */ +declare var MD: MD + /** * JSONL parsing and stringifying. */