Merge pull request #16 from JasperVanDenBosch/skip-gracefully

skipping pages if template or content is missing

.github/workflows/test.yml

@@ -0,0 +1,33 @@
+name: Test
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+
+  tests:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Python Version
+      run: python --version
+    - name: Update Pip
+      run: python -m pip install --upgrade pip
+    - name: Install dependencies
+      run: pip install -r requirements.txt
+    - name: Install Syrinx
+      run: pip install .
+    - name: Install test dependencies
+      run: pip install -r tests/requirements.txt
+    - name: Unit tests
+      run: pytest

meta/content/index.md

@@ -25,6 +25,10 @@ Organize your content with some standard structure and syrinx will interpret it.
 
 1. a `content/` directory with markdown file `index.md`
 2. *Toml* style frontmatter: Markdown files start with a standard header or *"frontmatter"* that is surrounded by triple pluses `+++`. 
+3. in content directories other than root, `index.md` is optional. Leaving it out signifies that you do not want a separate page build for this branch.
+4. Special frontmatter entries:
+    - `SequenceNumber`: Used by templates to order child items in menu's and lists.
+    - `Archetype`: The name of the template used to import these table data
 
 
 ### Templating and style

meta/dist/index.html

@@ -36,11 +36,17 @@ <h3>Content</h3>
 <ol>
 <li>a <code>content/</code> directory with markdown file <code>index.md</code></li>
 <li><em>Toml</em> style frontmatter: Markdown files start with a standard header or <em>"frontmatter"</em> that is surrounded by triple pluses <code>+++</code>. </li>
+<li>in content directories other than root, <code>index.md</code> is optional. Leaving it out signifies that you do not want a separate page build for this branch.</li>
+<li>Special frontmatter entries:<ul>
+<li><code>SequenceNumber</code>: Used by templates to order child items in menu's and lists.</li>
+<li><code>Archetype</code>: The name of the template used to import these table data</li>
+</ul>
+</li>
 </ol>
 <h3>Templating and style</h3>
 <ol>
 <li>Templates are written in <em>jinja2</em> and go into the <code>theme/templates/</code> directory.</li>
-<li>The default template used is <code>page.jinja2</code>. If a template is found matching the name of the node (<code>foo.jinja2</code>), or <code>root.jinja2</code> for the top-most page, this takes precedence.</li>
+<li>The default template used is <code>page.jinja2</code>. If a template is found matching the name of the node (e.g. <code>foo.jinja2</code>), or <code>root.jinja2</code> for the top-most page, this takes precedence.</li>
 <li>CSS, images and other assets go into <code>theme/assets/</code>.</li>
 </ol>
 <h3>Table Data</h3>

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "syrinx"
-version = "0.0.5"
+version = "0.0.6"
 authors = [
   { name="Jasper van den Bosch"},
 ]

requirements.txt

@@ -1,6 +1,4 @@
 jinja2
 markdown
 pandas
-pytest
-lxml
-parameterized
+lxml

syrinx/build.py

@@ -1,17 +1,24 @@
 from __future__ import annotations
-from typing import TYPE_CHECKING, Callable
+from typing import TYPE_CHECKING, Callable, Optional
 from os.path import isdir, join, isfile
 import shutil, os
 from jinja2 import Environment, FileSystemLoader, select_autoescape
+from syrinx.exceptions import ThemeError
 if TYPE_CHECKING:
     from syrinx.read import ContentNode
 
 
-def choose_template_file(node: ContentNode, isfile: Callable[[str], bool], dir: str) -> str:
+def choose_template_file(
+            node: ContentNode,
+            isfile: Callable[[str], bool],
+            dir: str
+        ) -> str:
     name = node.name or 'root'
-    if isfile(join(dir, f'{name}.jinja2')):
-        return f'{name}.jinja2'
-    return 'page.jinja2'
+    for tem_name in (name, 'page'):
+        if isfile(join(dir, f'{tem_name}.jinja2')):
+            return f'{tem_name}.jinja2'
+    else:
+        raise ThemeError(f'Missing template for "{node.name}"')
 
 
 def dir_exists_not_empty(path: str) -> bool:
@@ -21,6 +28,28 @@ def dir_exists_not_empty(path: str) -> bool:
     return False
 
 
+def build_node(
+        node: ContentNode,
+        root: ContentNode,
+        parent_path: str,
+        template_dir: str,
+        env: Environment
+    ):
+    """Recursive function to render page, then move on to children
+    """
+    if node.buildPage:
+        fname_tem = choose_template_file(node, isfile, template_dir)
+        page_template = env.get_template(fname_tem)
+        html = page_template.render(index=node, root=root)
+        node_path = join(parent_path, node.name)
+        os.makedirs(node_path, exist_ok=True)
+        out_fpath = join(node_path, 'index.html')
+        with open(out_fpath, 'w') as fhandle:
+            fhandle.write(html)
+    for child in node.branches:
+        build_node(child, root, node_path, template_dir, env)
+
+
 def build(root: ContentNode, root_dir: str):
 
     assert isdir(root_dir)
@@ -38,20 +67,7 @@ def build(root: ContentNode, root_dir: str):
         shutil.rmtree(dist_dir)
     os.makedirs(dist_dir, exist_ok=True)
 
-    def build_node(node: ContentNode, root: ContentNode, parent_path: str):
-        fname_tem = choose_template_file(node, isfile, template_dir)
-        page_template = env.get_template(fname_tem)
-        html = page_template.render(index=node, root=root)
-        node_path = join(parent_path, node.name)
-        os.makedirs(node_path, exist_ok=True)
-        out_fpath = join(node_path, 'index.html')
-        with open(out_fpath, 'w') as fhandle:
-            fhandle.write(html)
-        for child in node.branches:
-            build_node(child, root, node_path)
-
-
-    build_node(root, root, dist_dir)
+    build_node(root, root, dist_dir, template_dir, env)
 
     dist_assets_dir = join(dist_dir, 'assets')
 

syrinx/exceptions.py

@@ -0,0 +1,7 @@
+
+class ContentError(Exception):
+    pass
+
+
+class ThemeError(Exception):
+    pass

syrinx/preprocess.py

@@ -8,7 +8,7 @@
 """
 from __future__ import annotations
 from typing import TYPE_CHECKING, Dict
-from os.path import join, isdir, abspath, basename
+from os.path import join, isdir, basename
 from os import makedirs
 from glob import glob
 from jinja2 import Environment, FileSystemLoader, select_autoescape
@@ -20,7 +20,6 @@
 
 def preprocess(root_dir: str) -> None:
 
-
     assert isdir(root_dir)
 
     archetype_files = glob(join(root_dir, 'archetypes', '*.md'))
@@ -58,6 +57,8 @@ def preprocess(root_dir: str) -> None:
         if 'SequenceNumber' not in df.columns:
             df['SequenceNumber'] = list(range(len(df)))
 
+        df['Archetype'] = [archetype_name]*len(df)
+
         for label, row in df.iterrows():
             output = archetype.render(dict(row) | {df.index.name: label})
             with open(join(collection_dir, f'{label}.md'), 'w') as fhandle:

syrinx/read.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
-from typing import List, Dict
+from typing import List, Dict, Tuple
 from os.path import dirname, basename, join
-import os
+from os import walk
 import tomllib
 from markdown import markdown
+from sys import maxsize as SYS_MAX_SIZE
+from syrinx.exceptions import ContentError
 
 """
 This section is just about reading and interpreting the content
@@ -16,6 +18,13 @@ class ContentNode:
     content_html: str
     front: Dict[str, str]
     sequenceNumber: int
+    buildPage: bool
+
+    def __init__(self):
+        self.buildPage = False
+        self.leaves = []
+        self.branches = []
+        self.sequenceNumber = SYS_MAX_SIZE
 
 def reorder_children(node: ContentNode):
     node.leaves = sorted(node.leaves, key=lambda n: (n.sequenceNumber, n.name))
@@ -24,46 +33,48 @@ def reorder_children(node: ContentNode):
         reorder_children(child)
 
 
+def read_file(fpath: str) -> Tuple[Dict, str]:
+    with open(fpath) as fhandle:
+        lines = fhandle.readlines()
+    markers = [l for (l, line) in enumerate(lines) if line.strip() == '+++']
+    assert len(markers) == 2
+    fm_string = ''.join(lines[1:markers[1]])
+    fm_dict = tomllib.loads(fm_string)
+    md_content = ''.join(lines[markers[1]+1:])
+    return fm_dict, md_content
+
+
 def read(root_dir: str) -> ContentNode:
 
-
     content_dir = join(root_dir, 'content')
 
-
     tree: Dict[str, ContentNode] = dict()
     root = ContentNode()
     root.name = ''
-    for (dirpath, dirnames, fnames) in os.walk(content_dir):
-
-        ## ideally process the index page first (not sure if this is necessary?)
-        index_index = fnames.index('index.md')
-        fnames.insert(0, fnames.pop(index_index))
+    for (dirpath, _, fnames) in walk(content_dir):
 
         indexNode = ContentNode()
         indexNode.name = basename(dirpath)
         if dirpath == content_dir:
             indexNode = root
+            if 'index.md' not in fnames:
+                raise ContentError('root index file missing') 
         else:
             parent = tree[dirname(dirpath)]
             parent.branches.append(indexNode)
         tree[dirpath] = indexNode
+
+        ## ideally process the index page first (not sure if this is necessary?)
+        if 'index.md' in fnames:
+            fnames.insert(0, fnames.pop(fnames.index('index.md')))
         for fname in fnames:
             fparts = fname.split('.')
             ext = fparts[-1]
             if ext != 'md':
                 continue
             name = fparts[0]
 
-            in_fpath = join(dirpath, fname)
-
-            with open(in_fpath) as fhandle:
-                lines = fhandle.readlines()
-
-            markers = [l for (l, line) in enumerate(lines) if line.strip() == '+++']
-            assert len(markers) == 2
-            fm_string = ''.join(lines[1:markers[1]])
-            fm_dict = tomllib.loads(fm_string)
-            md_content = ''.join(lines[markers[1]+1:])
+            fm_dict, md_content = read_file(join(dirpath, fname))
 
             if name == 'index':
                 node = indexNode
@@ -72,11 +83,11 @@ def read(root_dir: str) -> ContentNode:
                 node.name = name
                 indexNode.leaves.append(node)
 
-            node.leaves = []
-            node.branches = []
             node.front = fm_dict
             node.content_html = markdown(md_content)
-            node.sequenceNumber = int(fm_dict.get('SequenceNumber', '99999'))
+            if 'SequenceNumber' in fm_dict:
+                node.sequenceNumber = fm_dict['SequenceNumber']
+            node.buildPage = True
 
     reorder_children(root)
 

tests/build.py

@@ -33,3 +33,15 @@ def test_choose_template_file_other(self, foo_exists: bool, expected: str):
             choose_template_file(node, isfile, '/t'),
             expected
         )
+
+    def test_choose_template_file_none(self):
+        """If no template is available, throw exception
+        """
+        from syrinx.build import choose_template_file
+        from syrinx.exceptions import ThemeError
+        node = Mock()
+        isfile = Mock()
+        node.name = 'foo'
+        isfile.side_effect = lambda p: False
+        with self.assertRaisesRegex(ThemeError, 'Missing template for "foo"'):
+            choose_template_file(node, isfile, '/t')

tests/build_node.py

@@ -0,0 +1,21 @@
+from unittest import TestCase
+from unittest.mock import Mock, patch
+
+class BuildNodeTests(TestCase):
+
+    @patch('syrinx.build.isfile')
+    @patch('syrinx.build.open')
+    def test_follows_buildPage(self, open, isfile):
+        from syrinx.build import build_node
+        node = Mock()
+        root = Mock()
+        env = Mock()
+        isfile.return_value = True
+        node.name = 'foo'
+        node.branches = []
+        node.buildPage = False
+        build_node(node, root, '', '', env)
+        self.assertFalse(env.get_template().render.called)
+        node.buildPage = True
+        build_node(node, root, '', '', env)
+        self.assertTrue(env.get_template().render.called)

tests/read.py

@@ -0,0 +1,44 @@
+from unittest import TestCase
+from unittest.mock import patch
+
+
+class ReadTests(TestCase):
+
+    @patch('syrinx.read.walk')
+    @patch('syrinx.read.read_file')
+    def test_read_BuildPage_branch_with_index(self, read_file, walk):
+        """If index.md present on branch then set "BuildPage" to true
+        """
+        read_file.return_value = dict(), ''
+        walk.return_value = [
+            ('/pth/content', None, ['index.md']),
+            ('/pth/content/lorem', None, ['ipsum.md', 'index.md']),
+        ]
+        from syrinx.read import read
+        root = read('/pth')
+        self.assertTrue(root.branches[0].buildPage)
+
+    @patch('syrinx.read.walk')
+    @patch('syrinx.read.read_file')
+    def test_read_BuildPage_branch_without_index(self, read_file, walk):
+        """If index.md absent on branch then set "BuildPage" to false
+        """
+        read_file.return_value = dict(), ''
+        walk.return_value = [
+            ('/pth/content', None, ['index.md']),
+            ('/pth/content/foo', None, ['bar.md']),
+        ]
+        from syrinx.read import read
+        root = read('/pth')
+        self.assertFalse(root.branches[0].buildPage)
+
+    @patch('syrinx.read.walk')
+    def test_read_Fail_if_index_missing(self, walk):
+        """The root index.md is not optional,
+        raise an exception if it's missing.
+        """
+        walk.return_value = [('/pth/content', None, ['other.md'])]
+        from syrinx.read import read
+        from syrinx.exceptions import ContentError
+        with self.assertRaisesRegex(ContentError, 'root index file missing'):
+            read('/pth')

tests/requirements.txt

@@ -0,0 +1,2 @@
+pytest
+parameterized