wip: broken pipeline

.github/workflows/main.yml

@@ -22,17 +22,17 @@ jobs:
           path: til-db
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN_FOR_PUSH }}
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
-      - uses: actions/cache@v2
+          python-version: 3.11
+      - uses: actions/cache@v3
         name: Configure pip caching
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-
@@ -85,9 +85,9 @@ jobs:
           git commit --amend --no-edit
           git push --force
       - name: Setup Node.js
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
-          node-version: "12.x"
+          node-version: 18
       - name: Deploy Datasette using Heroku
         env:
           HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}

build_db.py

@@ -1,13 +1,12 @@
-from datetime import timezone
-import httpx
 import git
+import httpx
 import os
 import pathlib
-from urllib.parse import urlencode
 import sqlite_utils
-from sqlite_utils.db import NotFoundError
 import time
-from datasette.utils import tilde_encode
+
+from datetime import timezone
+from sqlite_utils.db import NotFoundError
 
 root = pathlib.Path(__file__).parent.resolve()
 
@@ -38,57 +37,70 @@ def build_database(repo_path):
     all_times = created_changed_times(repo_path)
     db = sqlite_utils.Database(repo_path / "til.db")
     table = db.table("til", pk="path")
+
     for filepath in root.glob("*/*.md"):
         fp = filepath.open()
         title = fp.readline().lstrip("#").strip()
         body = fp.read().strip()
         path = str(filepath.relative_to(root))
+        slug = filepath.stem
         url = "https://github.com/jthodge/til/blob/main/{}".format(path)
-        # Do we need to render the markdown?
         path_slug = path.replace("/", "_")
+        topic = path.split("/")[0]
+
         try:
             row = table.get(path_slug)
             previous_body = row["body"]
             previous_html = row["html"]
         except (NotFoundError, KeyError):
             previous_body = None
             previous_html = None
+
         record = {
             "path": path_slug,
-            "topic": path.split("/")[0],
+            "slug": slug,
+            "topic": topic,
             "title": title,
             "url": url,
             "body": body,
         }
+
         if (body != previous_body) or not previous_html:
             retries = 0
+            response = None
+
             while retries < 3:
                 headers = {}
                 if os.environ.get("GITHUB_TOKEN"):
                     headers = {
                         "authorization": "Bearer {}".format(os.environ["GITHUB_TOKEN"])
                     }
+
                 response = httpx.post(
                     "https://api.github.com/markdown",
                     json={
-                        # mode=gfm would expand #13 issue links and suchlike
                         "mode": "markdown",
                         "text": body,
                     },
                     headers=headers,
                 )
+
                 if response.status_code == 200:
                     record["html"] = response.text
                     print("Rendered HTML for {}".format(path))
                     break
+                elif response.status_code == 401:
+                    assert False, "401 Unauthorized returned from GitHub API when rendering markdown"
                 else:
+                    print(response.status_code, response.headers)
                     print("  sleeping 60s")
                     time.sleep(60)
                     retries += 1
             else:
                 assert False, "Could not render {} - last response was {}".format(
                     path, response.headers
                 )
+
         record.update(all_times[path])
         with db.conn:
             table.upsert(record, alter=True)

metadata.yaml

@@ -1,12 +1,18 @@
 title: Today I Learned
 about: jthodge/til
 about_url: https://github.com/jthodge/til
-# plugins:
-#   datasette-atom:
-#     allow_unsafe_html_in_canned_queries: true
-#   datasette-media:
-#     screenshot:
-#       "sql": "select shot as content, 'image/png' as content_type from til where path=:key"
+plugins:
+  datasette-graphql:
+    path: /-/graphql
+  datasette-atom:
+    allow_unsafe_html_in_canned_queries: true
+  datasette-block-robots:
+    literal: |-
+      User-agent: *
+      Disallow: /tils
+  datasette-sitemap:
+    sql: |-
+      select '/' || topic || '/' || slug as path from til
 databases:
   til:
     queries:
@@ -21,11 +27,11 @@ databases:
         where
           til_fts match case
             :q
-            when '' then '*'
+            when '' then 'nomatchforthisterm'
             else escape_fts(:q)
           end
         order by
-          til_fts.rank limit 20
+          til_fts.rank limit 40
       feed:
         title: Taylor Hodge - TIL
         sql: |-

update_readme.py

@@ -1,18 +1,22 @@
 "Run this after build_database.py - it needs til.db"
 import pathlib
+import re
 import sqlite_utils
 import sys
-import re
 
 root = pathlib.Path(__file__).parent.resolve()
 
 index_re = re.compile(r"<!\-\- index starts \-\->.*<!\-\- index ends \-\->", re.DOTALL)
+count_re = re.compile(r"<!\-\- count starts \-\->.*<!\-\- count ends \-\->", re.DOTALL)
+
+COUNT_TEMPLATE = "<!-- count starts -->{}<!-- count ends -->"
 
 if __name__ == "__main__":
     db = sqlite_utils.Database(root / "til.db")
     by_topic = {}
     for row in db["til"].rows_where(order_by="created_utc"):
         by_topic.setdefault(row["topic"], []).append(row)
+
     index = ["<!-- index starts -->"]
     for topic, rows in by_topic.items():
         index.append("## {}\n".format(topic))
@@ -23,14 +27,21 @@
                 )
             )
         index.append("")
+
     if index[-1] == "":
         index.pop()
+
     index.append("<!-- index ends -->")
+
     if "--rewrite" in sys.argv:
         readme = root / "README.md"
         index_txt = "\n".join(index).strip()
         readme_contents = readme.open().read()
-        readme.open("w").write(index_re.sub(index_txt, readme_contents))
+        rewritten = index_re.sub(index_txt, readme_contents)
+        rewritten = count_re.sub(
+            COUNT_TEMPLATE.format(db["til"].count), rewritten
+        )
+        readme.open("w").write(rewritten)
     else:
         print("\n".join(index))