Support for working with markdown front matter (#642)

* Support for working with markdown front matter

Fixes #641

Add support for working with markdown front matter.

* **packages/core/src/frontmatter.ts**
  - Add `splitMarkdown` function to extract frontmatter and markdown content.
  - Add `updateFrontmatter` function to update frontmatter with new content.
  - Ensure `splitMarkdown` and `updateFrontmatter` handle various formats (yaml, json, toml).

* **packages/core/src/frontmatter.test.ts**
  - Add tests for `splitMarkdown` function.
  - Add tests for `updateFrontmatter` function.

* **packages/core/src/markdown.ts**
  - Import `splitMarkdown` and `updateFrontmatter` from `frontmatter.ts`.
  - Add `mergeFrontmatter` function to split markdown and update frontmatter.
  - Ensure `mergeFrontmatter` handles various formats (yaml, json, toml).

---

For more details, open the [Copilot Workspace session](https://copilot-workspace.githubnext.com/microsoft/genaiscript/issues/641?shareId=XXXX-XXXX-XXXX-XXXX).

* removing toml support

* typecheck issue

* add global object

* add MD type

docs/src/content/docs/reference/scripts/md.md

@@ -0,0 +1,34 @@
+---
+title: Markdown
+sidebar:
+    order: 9.2
+keywords: markdown, mdx, frontmatter
+---
+
+The `MD` class contains several helpers to work with [Markdown](https://www.markdownguide.org/cheat-sheet/) and [frontmatter text](https://jekyllrb.com/docs/front-matter/).
+
+The parser also support markdown variants like [MDX](https://mdxjs.com/).
+
+## `frontmatter`
+
+Extracts and parses the frontmatter text from a markdown file. Returns `undefined` if no frontmatter is not found or the parsing failed. Default format is `yaml`.
+
+```javascript
+const frontmatter = MD.frontmatter(text, "yaml")
+```
+
+## `content`
+
+Extracts the markdown source without the frontmatter.
+
+```javascript
+const content = MD.content(text)
+```
+
+## `updateFrontmatter`
+
+Merges frontmatter values into the existing in a markdown file. Use `null` value to delete fields.
+
+```javascript
+const updated = MD.updateFrontmatter(text, { title: "New Title" })
+```

docs/src/content/docs/reference/scripts/parsers.md

@@ -129,7 +129,7 @@ title: "Hello, World!"
 ...
 ```
 
-You can use the `parsers.frontmatter` to parse out the metadata into an object
+You can use the `parsers.frontmatter` or [MD](./md.md) to parse out the metadata into an object
 
 ```js
 const meta = parsers.frontmatter(file)

genaisrc/blog-generator.genai.mjs

@@ -170,8 +170,15 @@ defOutputProcessor((output) => {
     if (/\`\`\`markdown\n/.test(md)) {
         md = md.replace(/\`\`\`markdown\n/g, "").replace(/\`\`\`\n?$/g, "")
     }
-    const fm = parsers.frontmatter(md)
+    const fm = MD.frontmatter(md)
     if (!fm) throw new Error("No frontmatter found")
+
+    md = MD.updateFrontmatter(md, {
+        draft: true,
+        date: formattedDate,
+        authors: "genaiscript",
+    })
+
     const fn =
         `docs/src/content/docs/blog/drafts/${fm.title.replace(/[^a-z0-9]+/gi, "-")}.md`.toLocaleLowerCase()
     const sn =

packages/core/src/annotations.test.ts

@@ -13,7 +13,7 @@ describe("annotations", () => {
         `
 
         const diags = parseAnnotations(output)
-        console.log(diags)
+       // console.log(diags)
         assert.strictEqual(diags.length, 3)
         assert.strictEqual(diags[0].severity, "error")
         assert.strictEqual(diags[0].filename, "packages/core/src/github.ts")

packages/core/src/frontmatter.test.ts

@@ -1,6 +1,11 @@
 import { describe, test } from "node:test"
 import assert from "node:assert/strict"
-import { frontmatterTryParse } from "./frontmatter"
+import {
+    frontmatterTryParse,
+    splitMarkdown,
+    updateFrontmatter,
+} from "./frontmatter"
+import { YAMLTryParse } from "./yaml"
 
 describe("replace frontmatter", () => {
     test("only", () => {
@@ -21,3 +26,59 @@ foo bar
         assert.deepEqual(res, { foo: "bar" })
     })
 })
+
+describe("splitMarkdown", () => {
+    test("split markdown with yaml frontmatter", () => {
+        const markdown = `---
+title: Test
+---
+This is a test.`
+        const { frontmatter, content } = splitMarkdown(markdown)
+        assert.deepEqual(YAMLTryParse(frontmatter), { title: "Test" })
+        assert.equal(content, "This is a test.")
+    })
+
+    test("split markdown with json frontmatter", () => {
+        const markdown = `---
+{
+  "title": "Test"
+}
+---
+This is a test.`
+        const { frontmatter, content } = splitMarkdown(markdown)
+        assert.deepEqual(JSON.parse(frontmatter), { title: "Test" })
+        assert.equal(content, "This is a test.")
+    })
+})
+
+describe("updateFrontmatter", () => {
+    test("update yaml frontmatter", () => {
+        const markdown = `---
+title: Old Title
+foo: bar
+---
+This is a test.`
+        const newFrontmatter: any = { title: "New Title", foo: null }
+        const updatedMarkdown = updateFrontmatter(markdown, newFrontmatter)
+        const { frontmatter, content } = splitMarkdown(updatedMarkdown)
+        assert.deepEqual(YAMLTryParse(frontmatter), { title: "New Title" })
+        assert.equal(content, "This is a test.")
+    })
+
+    test("update json frontmatter", () => {
+        const markdown = `---
+{
+  "title": "Old Title",
+"foo": "bar"
+}
+---
+This is a test.`
+        const newFrontmatter: any = { title: "New Title", foo: null }
+        const updatedMarkdown = updateFrontmatter(markdown, newFrontmatter, {
+            format: "json",
+        })
+        const { frontmatter, content } = splitMarkdown(updatedMarkdown)
+        assert.deepEqual(JSON.parse(frontmatter), { title: "New Title" })
+        assert.equal(content, "This is a test.")
+    })
+})

packages/core/src/frontmatter.ts

@@ -1,36 +1,84 @@
+import { objectFromMap } from "pdfjs-dist/types/src/shared/util"
 import { JSON5TryParse } from "./json5"
 import { TOMLTryParse } from "./toml"
-import { YAMLTryParse } from "./yaml"
+import { YAMLTryParse, YAMLStringify } from "./yaml"
 
 export function frontmatterTryParse(
     text: string,
-    options?: { format: "yaml" | "json" | "toml" }
-): { end: number; value: any } {
-    if (!text) return undefined
-
+    options?: { format: "yaml" | "json" | "toml" | "text" }
+): { text: string; value: any; endLine?: number } | undefined {
     const { format = "yaml" } = options || {}
+    const { frontmatter, endLine } = splitMarkdown(text)
+    if (!frontmatter) return undefined
+
+    let res: any
+    switch (format) {
+        case "text":
+            res = frontmatter
+            break
+        case "json":
+            res = JSON5TryParse(frontmatter)
+            break
+        case "toml":
+            res = TOMLTryParse(frontmatter)
+            break
+        default:
+            res = YAMLTryParse(frontmatter)
+            break
+    }
+    return { text: frontmatter, value: res, endLine }
+}
 
+export function splitMarkdown(text: string): {
+    frontmatter?: string
+    endLine?: number
+    content: string
+} {
+    if (!text) return { content: text }
     const lines = text.split(/\r?\n/g)
     const delimiter = "---"
-    if (lines[0] !== delimiter) return undefined
+    if (lines[0] !== delimiter) return { content: text }
     let end = 1
     while (end < lines.length) {
         if (lines[end] === delimiter) break
         end++
     }
-    if (end >= lines.length) return undefined
-    const fm = lines.slice(1, end).join("\n")
-    let res: any
+    if (end >= lines.length) return { content: text }
+    const frontmatter = lines.slice(1, end).join("\n")
+    const content = lines.slice(end + 1).join("\n")
+    return { frontmatter, content, endLine: end }
+}
+
+export function updateFrontmatter(
+    text: string,
+    newFrontmatter: any,
+    options?: { format: "yaml" | "json" }
+): string {
+    const { content = "" } = splitMarkdown(text)
+    if (newFrontmatter === null) return content
+
+    const frontmatter = frontmatterTryParse(text, options)?.value ?? {}
+
+    // merge object
+    for (const [key, value] of Object.entries(newFrontmatter ?? {})) {
+        if (value === null) {
+            delete frontmatter[key]
+        } else {
+            frontmatter[key] = value
+        }
+    }
+
+    const { format = "yaml" } = options || {}
+    let fm: string
     switch (format) {
         case "json":
-            res = JSON5TryParse(fm)
+            fm = JSON.stringify(frontmatter, null, 2)
             break
-        case "toml":
-            res = TOMLTryParse(fm)
+        case "yaml":
+            fm = YAMLStringify(frontmatter)
             break
         default:
-            res = YAMLTryParse(fm)
-            break
+            throw new Error(`Unsupported format: ${format}`)
     }
-    return res !== undefined ? { end: end + 1, value: res } : undefined
+    return `---\n${fm}\n---\n${content}`
 }