[add] new parser

docs/template.typ

@@ -1,4 +1,4 @@
-#import "@preview/codly:0.1.0": *
+#import "@preview/codly:1.0.0": codly-init, no-codly, codly
 
 // The project function defines how your document looks.
 // It takes your content and some metadata and formats it.
@@ -16,7 +16,6 @@
   // Set the document's basic properties.
   set document(author: authors, title: title)
   set page(numbering: "1", number-align: center)
-  set text(font: "Linux Libertine", lang: "en")
 
   show heading.where(level: 1): it => block(smallcaps(it), below: 1em)
   // set heading(numbering: (..args) => if args.pos().len() == 1 { numbering("I", ..args) })
@@ -72,12 +71,16 @@
   v(10em)
 
 
-  show: codly-init
+  show: codly-init.with(
+  )
   codly(
-    languages: (
-      // typ: (name: "typ", icon: none, color: rgb("#239DAE")),
-    ),
+    fill: white
   )
+  // codly(
+  //   languages: (:
+  //     // typ: (name: "typ", icon: none, color: rgb("#239DAE")),
+  //   ),
+  // )
   show raw.where(block: true): set text(size: .95em)
   show raw.where(block: true): it => pad(x: 4%, it)
   show raw.where(block: false, lang: "typ").or(raw.where(lang: "notnone")): it => box(inset: (x: 3pt), outset: (y: 3pt), radius: 40%, fill: luma(235), it)
@@ -103,10 +106,9 @@
 ))
 
 
-#let tidy-output-figure(output) = {
+#let tidy-output-figure(output) = no-codly({
   set heading(numbering: none)
   set text(size: .8em)
-  disable-codly()
   figure(align(left, box(
     width: 80%,
     stroke: 0.5pt + luma(200), 
@@ -117,5 +119,4 @@
       output
     )
   )))
-  codly()
-}
+})

docs/tidy-guide.typ

@@ -1,5 +1,6 @@
 #import "template.typ": *
 #import "/src/tidy.typ"
+#include "/tests/parser/test.typ"
 #include "/tests/test_tidy.typ" // ensure that tests pass
 
 #let version = toml("/typst.toml").package.version
@@ -43,28 +44,38 @@ We now assume we have a Typst module called `repeater.typ`, containing a definit
 #let example-code = read("/examples/repeater.typ")
 #file-code("repeater.typ", raw(block: true, lang: "typ", example-code))
 
-A *function* is documented similar to javadoc by prepending a block of `///` comments. Each line needs to start with three slashes `///` (whitespace is allowed at the beginning of the line). _Parameters_ of the function can be documented by listing them as 
-#show raw.where(lang: "markspace"): it => {
-  show " ": box(inset: (x: 0.1pt), box(
-    fill: red.lighten(70%), 
-    width: .7em, height: .8em,
-    radius: 1pt,
-    outset: (bottom: 3pt, top: 1pt),
-  ))
-  it
-}
-```markspace
-/// - parameter-name (type): ...
-```
-Following this exact form is important (see also the spaces marked in red) since this allows to distinguish the parameter list from ordinary markup lists in the function description or in parameter descriptions. For example, another space in front of the `-` could be added to markup lists if necessary. 
+Tidy uses `///` doc-comments for documentation. 
+A function or variable can be provided with a *description* by placing a doc-comment just before the definition. 
+
+Until type annotations are natively available in Typst, a return type can be annotated with the `->` syntax in the last line of the description. If more there is more than one possible return type, they can be given separated by the pipe `|` operator, e.g., `-> int, float`. 
+
+Function arguments are documented in the same way. 
+All descriptions are parsed as Typst markup. See @user-defined-symbols on how to add images or examples to a description. 
 
-The possible types for each parameter are given in parentheses and after a colon `:`, the parameter description follows. Indicating a type is mandatory (you may want to pick `any` in some cases). An optional _return type_ can be annotated by ending with a line that contains `->` followed by the return type(s). 
 
-In front of the parameter list, a _function description_ can be put. Both function and parameter descriptions may span multiple lines and can contain any Typst code (see @user-defined-symbols on how to use images, user-defined variables and functions in the docstring). 
+// A *function* is documented vi 
+// similar to javadoc by prepending a block of `///` comments. Each line needs to start with three slashes `///` (whitespace is allowed at the beginning of the line). _Parameters_ of the function can be documented by listing them as 
+// #show raw.where(lang: "markspace"): it => {
+//   show " ": box(inset: (x: 0.1pt), box(
+//     fill: red.lighten(70%), 
+//     width: .7em, height: .8em,
+//     radius: 1pt,
+//     outset: (bottom: 3pt, top: 1pt),
+//   ))
+//   it
+// }
+// ```markspace
+// /// - parameter-name (type): ...
+// ```
+// Following this exact form is important (see also the spaces marked in red) since this allows to distinguish the parameter list from ordinary markup lists in the function description or in parameter descriptions. For example, another space in front of the `-` could be added to markup lists if necessary. 
 
-*Variables* are documented just in the same way (lacking the option to specify parameters). A definition is recognized as a variable if the identifier (variable/function name) is not followed by an opening parenthesis. The `->` syntax which also specifies the return type for functions can be used to define the type of a variable. 
+// The possible types for each parameter are given in parentheses and after a colon `:`, the parameter description follows. Indicating a type is mandatory (you may want to pick `any` in some cases). An optional _return type_ can be annotated by ending with a line that contains `->` followed by the return type(s). 
 
-Calling #ref-fn("parse-module()") will read out the documentation of the given string. We can then invoke #ref-fn("show-module()") on the result.
+// In front of the parameter list, a _function description_ can be put. 
+
+// *Variables* are documented just in the same way (lacking the option to specify parameters). A definition is recognized as a variable if the identifier (variable/function name) is not followed by an opening parenthesis. The `->` syntax which also specifies the return type for functions can be used to define the type of a variable. 
+
+Calling #ref-fn("parse-module()") will read out the documentation of the given string. We can then invoke #ref-fn("show-module()") on the returned docs object. The actual output depends on the utilized style template, see @customizing. 
 
 ```typ
 #let docs = tidy.parse-module(read("docs.typ"), name: "Repeater")
@@ -73,7 +84,7 @@ Calling #ref-fn("parse-module()") will read out the documentation of the given s
 
 This will produce the following output. 
 #tidy-output-figure(
-  tidy.show-module(tidy.parse-module(example-code, name: "Repeater"), style: tidy.styles.default)
+  tidy.show-module(tidy.parse-module(example-code, name: "Repeater", old-parser: false), style: tidy.styles.default)
 )
 
 
@@ -141,7 +152,8 @@ In the output, the preview of the code examples is shown next to it.
     read("/examples/wiggly.typ"), 
     name: "wiggly",
     scope: (wiggly: wiggly),
-    preamble: "import wiggly: draw-sine;"
+    preamble: "import wiggly: draw-sine;", 
+    old-parser: false
   )
   tidy-output-figure(tidy.show-module(module, show-outline: false))
 }
@@ -168,7 +180,8 @@ The function `example()` is available in every docstring and has some bells and
 
   let module = tidy.parse-module(
     read("/examples/example-demo.typ"), 
-    scope: (example-demo: example-demo)
+    scope: (example-demo: example-demo),
+    old-parser: false
   )
   tidy-output-figure(tidy.show-module(module, show-outline: false, break-param-descriptions: true))
 }
@@ -178,7 +191,7 @@ The function `example()` is available in every docstring and has some bells and
 
 
 
-= Customizing the style
+= Customizing the style <customizing>
 
 There are multiple ways to customize the output style. You can
 - pick a different predefined style,
@@ -219,9 +232,12 @@ With a dark background and light text, these colors produce much better contrast
   let module = tidy.parse-module(
     ```
     /// Produces space. 
-    /// - amount (length):
-    #let space(amount)
-    ```.text
+    #let space(
+      /// -> length
+      amount
+    )
+    ```.text,
+    old-parser: false
   )
   tidy-output-figure(tidy.show-module(module, show-outline: false, colors: tidy.styles.default.colors-dark, style: tidy.styles.default))
 }
@@ -239,7 +255,8 @@ Currently, the two predefined styles `tidy.styles.default` and `tidy-styles.mini
     read("/examples/wiggly.typ"), 
     name: "wiggly",
     scope: (wiggly: wiggly),
-    preamble: "import wiggly: *;"
+    preamble: "import wiggly: *;",
+    old-parser: false
   )
   tidy-output-figure(tidy.show-module(module, show-outline: false, style: tidy.styles.minimal))
 }
@@ -342,9 +359,12 @@ The default style for help output should work more or less for light and dark do
 
 When set up in the form as shown above, the package `tidy` is only imported when a user calls `help` for the first time and not at all if the feature is not used _(don't pay for what you don't use)_. The files themselves are also only read when a definition from a specific submodule in the "namespace" is requested. In the case of _extremely_ long code files, it _could_ make sense to separate the documentation from the implementation by adding "documentation files" that only contain a _declaration_ plus docstring for each definition -- with the body left empty. 
 ```typ
-/// - inputs (array): The inputs for the algorithm. 
-/// - parameters (none, dictionary): Some parameters. 
-#let my-really-long-algorithm(inputs, parameters: none) = { }
+#let my-really-long-algorithm(
+  /// The inputs for the algorithm. -> array
+  inputs, 
+  /// Some parameters. -> none | dictionary
+  parameters: none
+  ) = { }
 ```
 
 The advantage is that the source code is not as crowded with (sometimes very long) docstrings and that docstring parsing may get faster. On the downside, there is an increased maintenance overhead due to the need of synchronizing the actual file and the documentation file (especially when the interface of a function changes). 
@@ -412,7 +432,8 @@ Let us now "self-document" this package:
       read("/src/helping.typ")
     ).join("\n"),
     name: "tidy", 
-    require-all-parameters: true
+    require-all-parameters: true, 
+    old-parser: false
   )
   tidy.show-module(
     module, 

examples/repeater.typ

@@ -1,10 +1,15 @@
 /// Repeats content a specified number of times. 
-/// - body (content): The content to repeat. 
-/// - num (int):  Number of times to repeat the content. 
-/// - separator (content): Optional separator between repetitions 
-///                        of the content. 
 /// -> content
-#let repeat(body, num, separator: []) = ((body,)*num).join(separator)
+#let repeat(
+  /// The content to repeat. -> content
+  body, 
+
+  ///  Number of times to repeat the content. -> int
+  num, 
+
+  /// Optional separator between repetitions of the content. -> content
+  separator: []
+) = ((body,)*num).join(separator)
 
 /// An awfully bad approximation of pi. 
 /// -> float

examples/wiggly.typ

@@ -3,13 +3,21 @@
 /// *Example:*
 /// #example(`draw-sine(1cm, 0.5cm, 2)`)
 ///
-/// - height (length): Width of bounding rectangle.
-/// - width (length): Height of bounding rectangle.
-/// - periods (int, float): Number of periods to draw. 
-///      Example with many periods: 
-///      #example(`draw-sine(4cm, 1.3cm, 10)`)
 /// -> content
-#let draw-sine(width, height, periods) = box(width: width, height: height, {
+#let draw-sine(
+
+  /// Height of bounding rectangle. -> length
+  width, 
+
+  /// Width of bounding rectangle. -> length
+  height, 
+
+  /// Number of periods to draw. 
+  ///      Example with many periods: 
+  ///      #example(`draw-sine(4cm, 1.3cm, 10)`)
+  /// -> int | float
+  periods
+) = box(width: width, height: height, {
   let resolution = 100
   let frequency = 1 / resolution * 2 * calc.pi * periods
   let prev-point = (0pt, height / 2)

src/helping.typ

@@ -15,15 +15,15 @@
   )
 }
 
-#let parse-namespace-modules(entry) = {
+#let parse-namespace-modules(entry, old-parser: false) = {
   // "Module" is made up of several files
   if type(entry) != array {
     entry = (entry,)
   }
-  parse-module(entry.map(x => x()).join("\n"))
+  parse-module(entry.map(x => x()).join("\n"), old-parser: old-parser)
 }
 
-#let search-docs(search, searching, namespace, style) = {
+#let search-docs(search, searching, namespace, style, old-parser: false) = {
   if search == "" { return help-box(block[_empty search string_]) }
   let search-names = "n" in searching
   let search-descriptions = "d" in searching
@@ -42,11 +42,11 @@
   }
 
   let definitions = ()
-  let module = parse-namespace-modules(namespace.at("."))
+  let module = parse-namespace-modules(namespace.at("."), old-parser: old-parser)
   let functions = ()
   let variables = ()
   for (name, modules) in namespace {
-    let module = parse-namespace-modules(modules)
+    let module = parse-namespace-modules(modules, old-parser: old-parser)
 
     functions += module.functions.filter(filter)
     variables += module.variables.filter(x => search in x.name or search in x.description)
@@ -183,42 +183,53 @@
 /// Generates a `help` function for your package that allows the user to
 /// prints references directly into their document while typing. This allows
 /// them to easily check the usage and documentation of a function or variable. 
-///
-/// - namespace (dictionary): This dictionary should reflect the "namespace" of the package
-///        in a flat dictionary and contain `read.with()` instances for the respective code 
-///        files. 
-///        Imagine importing everything from a package, `#import "mypack.typ": *`. How a 
-///        symbol is accessible now determines how the dictionary should be built. 
-///        We start with a root key, `(".": read.with("lib.typ"))`. If `lib.typ` imports 
-///        symbols from other files _into_ its scope, these files should be added to the
-///        root along with `lib.typ` by passing an array: 
-///        ```typ
-///        (
-///          ".": (read.with("lib.typ"), read.with("more.typ")),
-///          "testing": read.with("testing.typ")
-///        )
-///        ```
-///        Here, we already show another case: let `testing.typ` be imported in `lib.typ`
-///        but without `*`, so that the symbols are accessed via `testing.`. We therefore 
-///        add these under a new key. Nested files should be added with multiple 
-///        dots, e.g., `"testing.float."`. 
-///        
-///        By providing instances of `read()` with the filename prepended, you allow tidy 
-///        to read the files that are not part of the tidy package but at the same time
-///        enable lazy evaluation of the files, i.e., a file is only opened when a 
-///        definition from this file is requested through `help()`. 
-/// - style (dictionary): A tidy style that is used for showing parts of the documentation
-///        in the help box. It is recommended to leave this at the `help` style which is 
-///        particularly designed for this purpose. Please post an issue if you have problems
-///        or suggestions regarding this style. 
-/// - package-name (str): The name of the package. This is required to give helpful
-///   error messages when a symbol cannot be found. 
-/// - onerror (function): What to do with errors. By default, an assertion is failed (the document panics). 
 #let generate-help(
+
+  /// This dictionary should reflect the "namespace" of the package
+  /// in a flat dictionary and contain `read.with()` instances for the respective code 
+  /// files. 
+  /// Imagine importing everything from a package, `#import "mypack.typ": *`. How a 
+  /// symbol is accessible now determines how the dictionary should be built. 
+  /// We start with a root key, `(".": read.with("lib.typ"))`. If `lib.typ` imports 
+  /// symbols from other files _into_ its scope, these files should be added to the
+  /// root along with `lib.typ` by passing an array: 
+  /// ```typ
+  /// (
+  ///   ".": (read.with("lib.typ"), read.with("more.typ")),
+  ///   "testing": read.with("testing.typ")
+  /// )
+  /// ```
+  /// Here, we already show another case: let `testing.typ` be imported in `lib.typ`
+  /// but without `*`, so that the symbols are accessed via `testing.`. We therefore 
+  /// add these under a new key. Nested files should be added with multiple 
+  /// dots, e.g., `"testing.float."`. 
+  /// 
+  /// By providing instances of `read()` with the filename prepended, you allow tidy 
+  /// to read the files that are not part of the tidy package but at the same time
+  /// enable lazy evaluation of the files, i.e., a file is only opened when a 
+  /// definition from this file is requested through `help()`. 
+  /// -> dictionary
   namespace: (".": () => ""), 
+
+  /// The name of the package. This is required to give helpful error messages when
+  ///  a symbol cannot be found. 
+  /// -> str
   package-name: "",
+
+  /// A tidy style that is used for showing parts of the documentation
+  /// in the help box. It is recommended to leave this at the `help` style which is 
+  /// particularly designed for this purpose. Please post an issue if you have problems
+  /// or suggestions regarding this style. 
+  /// -> dictionary
   style: styles.help,
-  onerror: msg => assert(false, message: msg)
+
+  /// What to do with errors. By default, an assertion is failed (the document panics). 
+  /// -> function
+  onerror: msg => assert(false, message: msg),
+
+  /// Whether to use the old parser. 
+  /// -> boolean
+  old-parser: false
 ) = {
 
   let validate-namespace-tree(namespace) = {
@@ -253,7 +264,7 @@
       let name = args.pos().first()
       help-box(get-docs(name, namespace, package-name, style, onerror: onerror))
     } else {
-      search-docs(search, searching, namespace, style)
+      search-docs(search, searching, namespace, style, old-parser: old-parser)
     }
   }
   help-function