upgrade to astro 5

bin/build-content-collection.js

@@ -5,54 +5,54 @@ import path from 'path';
 
 
 (async () => {
-async function buildContentCollection() {
-  // go through all files in .cache, move them to src/content
-  console.log('Building content collection');
+  async function buildContentCollection() {
+    // go through all files in .cache, move them to src/content
+    console.log('Building content collection');
 
-  const getAllMDFiles = (dir) =>
-    readdirSync(dir).reduce((files, file) => {
-      const name = path.join(dir, file);
-      const isDirectory = statSync(name).isDirectory();
-      if (isDirectory) {
-        return [...files, ...getAllMDFiles(name)];
-      } else {
-        if (/\.mdx?$/.test(name)) {
-          return [...files, name];
+    const getAllMDFiles = (dir) =>
+      readdirSync(dir).reduce((files, file) => {
+        const name = path.join(dir, file);
+        const isDirectory = statSync(name).isDirectory();
+        if (isDirectory) {
+          return [...files, ...getAllMDFiles(name)];
+        } else {
+          if (/\.mdx?$/.test(name)) {
+            return [...files, name];
+          }
+          return files;
         }
-        return files;
-      }
-    }, []);
+      }, []);
 
-      const files = getAllMDFiles('.cache');
-      const targetPath = path.resolve().endsWith('sites/pipelines')
-        ? 'src/content/pipelines'
-        : 'sites/pipelines/src/content/pipelines';
-      console.log('target path:', targetPath);
-      if (!existsSync(targetPath)) {
-        mkdirSync(targetPath, { recursive: true });
-        console.log('Created ', targetPath, ' folder');
-      }
-      Promise.all(
-        // create sites/pipelines/src/content/pipelines folder if it doesn't exist
+    const files = getAllMDFiles('.cache');
+    const targetPath = path.resolve().endsWith('sites/pipelines')
+      ? 'src/content/pipelines'
+      : 'sites/pipelines/src/content/pipelines';
+    console.log('target path:', targetPath);
+    if (!existsSync(targetPath)) {
+      mkdirSync(targetPath, { recursive: true });
+      console.log('Created ', targetPath, ' folder');
+    }
+    Promise.all(
+      // create sites/pipelines/src/content/pipelines folder if it doesn't exist
 
-        files.map(async (f) => {
-          let content = await promises.readFile(f, 'utf8');
-          const pathParts = f.split('/');
-          const pipeline = pathParts[1];
-          const version = pathParts[2];
-          // make relative links to png and svg files absolute in markdown to current Astro.url.pathname
-          content = content.replaceAll(
-            /(\]\()(docs\/images\/.*?\.png|svg)/gmu,
-            `$1${`https://raw.githubusercontent.com/nf-core/${pipeline}/${version}/$2`}`,
-          );
-          const newPath = f.replace('.cache', targetPath);
-          const parent = newPath.split('/').slice(0, -1).join('/');
-          await promises.mkdir(parent, { recursive: true });
-          await promises.writeFile(newPath, content);
-        }),
-      );
-  return true;
-};
+      files.map(async (f) => {
+        let content = await promises.readFile(f, 'utf8');
+        const pathParts = f.split('/');
+        const pipeline = pathParts[1];
+        const version = pathParts[2];
+        // make relative links to png and svg files absolute in markdown to current Astro.url.pathname
+        content = content.replaceAll(
+          /(\]\()(docs\/images\/.*?\.png|svg)/gmu,
+          `$1${`https://raw.githubusercontent.com/nf-core/${pipeline}/${version}/$2`}`,
+        );
+        const newPath = f.replace('.cache', targetPath);
+        const parent = newPath.split('/').slice(0, -1).join('/');
+        await promises.mkdir(parent, { recursive: true });
+        await promises.writeFile(newPath, content);
+      }),
+    );
+    return true;
+  }
 
-await buildContentCollection();
+  // await buildContentCollection();
 })();

sites/configs/tsconfig.json

@@ -1,5 +1,6 @@
 {
     "extends": "astro/tsconfigs/base",
+    "include": [".astro/types.d.ts", "**/*"],
     "compilerOptions": {
         "baseUrl": ".",
         "paths": {
@@ -15,8 +16,5 @@
         },
         "strictNullChecks": true
     },
-    "exclude": [
-        "node_modules",
-        "dist",
-    ]
+    "exclude": ["node_modules", "dist"]
 }

sites/docs/package.json

@@ -14,7 +14,8 @@
         "test": "playwright test"
     },
     "dependencies": {
-        "@astrojs/mdx": "^3.1.3",
+        "astro": "5.0.0-beta.3",
+        "@astrojs/mdx": "4.0.0-beta.1",
         "@astrojs/netlify": "^5.4.0",
         "@iconify-json/fa": "^1.1.8",
         "@iconify-json/fa-brands": "^1.1.8",

sites/docs/tsconfig.json

@@ -1,5 +1,6 @@
 {
     "extends": "astro/tsconfigs/base",
+    "include": [".astro/types.d.ts", "**/*"],
     "compilerOptions": {
         "baseUrl": ".",
         "paths": {
@@ -15,8 +16,5 @@
         },
         "strictNullChecks": true
     },
-    "exclude": [
-        "node_modules",
-        "dist",
-    ]
+    "exclude": ["node_modules", "dist"]
 }

sites/main-site/astro.config.mjs

@@ -35,7 +35,7 @@ pipelines_json.remote_workflows.map((pipeline) => {
 // https://astro.build/config
 export default defineConfig({
     site: 'https://nf-co.re/',
-    output: 'hybrid',
+    output: 'static',
     adapter: netlify(),
     prefetch: false,
     redirects: {
@@ -82,7 +82,7 @@ export default defineConfig({
                     'tools-16',
                 ],
                 'simple-icons': ['bluesky'],
-                'ri': ['open-source-line'],
+                ri: ['open-source-line'],
             },
         }),
         sitemap(),

sites/main-site/package.json

@@ -13,10 +13,10 @@
         "test": "playwright test"
     },
     "dependencies": {
-        "@astro-community/astro-embed-youtube": "^0.5.2",
+        "@astro-community/astro-embed-youtube": "^0.5.3",
         "@astrojs/internal-helpers": "^0.4.1",
         "@astrojs/markdown-remark": "^5.2.0",
-        "@astrojs/mdx": "^3.1.3",
+        "@astrojs/mdx": "4.0.0-beta.1",
         "@astrojs/netlify": "^5.4.0",
         "@astrojs/partytown": "^2.1.1",
         "@astrojs/rss": "^4.0.7",
@@ -38,7 +38,7 @@
         "@iconify/utils": "^2.1.25",
         "@nanostores/persistent": "^0.10.1",
         "@octokit/types": "^13.5.0",
-        "astro": "^4.12.2",
+        "astro": "5.0.0-beta.3",
         "astro-icon": "^1.1.0",
         "astro-remark-description": "^1.1.2",
         "bootstrap": "^5.3.3",
@@ -84,12 +84,11 @@
         "satori-html": "^0.3.2",
         "sharp": "^0.33.4",
         "shiki": "^1.11.0",
-        "shikiji": "^0.10.2",
         "string-width": "^7.2.0",
         "svelte": "^4.2.18",
         "svelte-confetti": "^1.4.0",
         "svelte-exmarkdown": "^3.0.5",
-        "svelte-fast-marquee": "^0.7.0",
+        "svelte-fast-marquee": "^0.8.0",
         "ts-dedent": "^2.2.0",
         "typescript": "^5.5.3",
         "unist-util-visit": "^5.0.0",

sites/main-site/src/components/BaseHead.astro

@@ -1,5 +1,5 @@
 ---
-import { ViewTransitions } from 'astro:transitions';
+import { ClientRouter } from 'astro:transitions';
 export interface Props {
     title: string;
     description: string;
@@ -52,7 +52,7 @@ const { title, description, image = '/social_img.png', docSearchTags = [], viewT
 <meta property="twitter:description" content={description} />
 <meta property="twitter:image" content={new URL(image, Astro.url)} />
 
-{viewTransitions && <ViewTransitions fallback="none" />}
+{viewTransitions && <ClientRouter fallback="none" />}
 
 <!-- DocSearch -->
 {docSearchTags.map(({ name, content }) => <meta name={'docsearch:' + name} content={content} />)}

sites/main-site/src/content/config.ts

@@ -1,5 +1,7 @@
 import { z, defineCollection } from 'astro:content';
+import { glob } from 'astro/loaders';
 
+import { pipelinesLoader } from '../loaders/pipelines.ts';
 const events = defineCollection({
     type: 'content',
     schema: z
@@ -98,7 +100,7 @@ const about = defineCollection({
 });
 
 const blog = defineCollection({
-    type: 'content',
+    loader: glob({ pattern: '**/*.{md,mdx}', base: './src/content/blog' }),
     schema: z
         .object({
             title: z.string(),
@@ -174,7 +176,9 @@ const specialInterestGroups = defineCollection({
         }),
 });
 
-const pipelines = defineCollection({});
+const pipelines = defineCollection({
+    loader: pipelinesLoader,
+});
 
 const api_reference = defineCollection({});
 

sites/main-site/src/loaders/pipelines.ts

@@ -0,0 +1,130 @@
+import type { Loader } from 'astro/loaders';
+import pipelines_json from '@public/pipelines.json';
+import { createMarkdownProcessor } from '@astrojs/markdown-remark';
+
+export const pipelinesLoader: Loader = {
+    name: 'pipelines',
+    load: async ({ store, logger, config, generateDigest }) => {
+        const processor = await createMarkdownProcessor(config.markdown);
+        // store.clear();
+        pipelines_json.remote_workflows.map(async (pipeline) => {
+            logger.info(`Loaded pipeline: ${pipeline.name}`);
+            if (pipeline.archived) {
+                return;
+            }
+            pipeline.releases.map(async (version) => {
+                const paths = ['README.md', ...version.doc_files];
+                paths.map(async (path) => {
+                    const repo = pipeline.name;
+                    const ref = version.tag_name;
+
+                    if (store.get(pipeline.name + '-' + ref + '-' + path)) {
+                        logger.info(`Skipping ${pipeline.name}-${ref}-${path}`);
+                        return;
+                    }
+                    logger.info(`fetching file: ${ref}=${path}`);
+                    const readmeUrl = `https://raw.githubusercontent.com/nf-core/${repo}/${ref}/${path}`;
+                    const response = await fetch(readmeUrl).catch((error) => {
+                        logger.error(`Error fetching ${readmeUrl}: ${error.message}`);
+                    });
+                    let content = await response?.text();
+                    if (!content) {
+                        return;
+                    }
+                    const parent_directory = path.split('/').slice(0, -1).join('/');
+                    // add github url to image links in markdown if they are relative
+                    content = content.replaceAll(/!\[([^\][]*[?[^\][]*\]?[^\][]*)\]\((.*?)\)/g, (match, p1, p2) => {
+                        if (p2.startsWith('http')) {
+                            return match;
+                        } else {
+                            return `![${p1}](https://raw.githubusercontent.com/nf-core/${repo}/${ref}/${parent_directory}/${p2})`;
+                        }
+                    });
+                    // add github url to html img tags in markdown
+                    content = content.replaceAll(/<img(.*?)src="(.*?)"/g, (match, p1, p2) => {
+                        if (p2.startsWith('http')) {
+                            return match;
+                        } else {
+                            return `<img${p1}src="https://raw.githubusercontent.com/nf-core/${repo}/${ref}/${parent_directory}/${p2}"`;
+                        }
+                    });
+                    // add github url to html img tags in markdown for dark mode images
+                    content = content.replaceAll(/<source(.*?)srcset="(.*?)"/g, (match, p1, p2) => {
+                        if (p2.startsWith('http')) {
+                            return match;
+                        } else {
+                            return `<source${p1}src="https://raw.githubusercontent.com/nf-core/${repo}/${ref}/${parent_directory}/${p2}"`;
+                        }
+                    });
+                    // prefix links to CONTRIBUTING.md, CITATIONS.md, CHANGELOG.md with github url
+                    content = content.replaceAll(
+                        /\[(.*?)\]\((\.github\/CONTRIBUTING\.md|CITATIONS\.md|CHANGELOG\.md)\)/g,
+                        (match, p1, p2) => {
+                            if (p2.startsWith('http')) {
+                                return match;
+                            } else {
+                                return `[${p1}](https://github.com/nf-core/${repo}/blob/${ref}/${p2})`;
+                            }
+                        },
+                    );
+                    // prefix links to files in the assets directory with github url
+                    content = content.replaceAll(/\[(.*?)\]\(((\.\.\/)*assets\/.*?)\)/g, (match, p1, p2) => {
+                        if (p2.startsWith('http')) {
+                            return match;
+                        } else {
+                            return `[${p1}](https://github.com/nf-core/${repo}/blob/${ref}/${p2.replace('../assets/', 'assets/')})`;
+                        }
+                    });
+
+                    // convert github style admonitions to docusaurus admonitions
+                    content = content.replace(
+                        /> \[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\]\s*\n((?:> [^\n]*\s*?)+)/g,
+                        (match, type, content) => {
+                            const cleanedContent = content.replace(/> /g, '').trim();
+                            const admonitionType = type.toLowerCase();
+
+                            if (admonitionType === 'important') {
+                                return `:::info{title=Important}\n${cleanedContent}\n:::\n\n`;
+                            }
+                            if (admonitionType === 'caution') {
+                                return `:::danger{title=Caution}\n${cleanedContent}\n:::\n\n`;
+                            }
+
+                            return `:::${admonitionType}\n${cleanedContent}\n:::\n\n`;
+                        },
+                    );
+
+                    // remove .md(x) from links with anchor tags
+                    content = content.replaceAll(/\[([^\][]*)\]\((.*?)\.mdx?#(.*?)\)/g, '[$1]($2#$3)');
+
+                    // remove github warning and everything before from docs
+                    content = content.replace(/(.*?)(## :warning:)(.*?)usage\)/s, '');
+                    // remove blockquote ending in "files._" from the start of the document
+                    content = content.replace(/(.*?)(files\._)/s, '');
+                    // cleanup heading
+                    content = content.replace('# nf-core/' + repo + ': ', '# ');
+                    // remove everything before introduction
+                    content = content.replace(/.*?## Introduction/gs, '## Introduction');
+                    // replace nextflow with groovy code blocks TODO: remove this when we have a nextflow syntax highlighter works in shiki
+                    content = content.replace(/```nextflow/g, '```groovy');
+                    const renderedContent = await processor.render(content);
+
+                    store.set({
+                        id: pipeline.name + '-' + ref + '-' + path,
+                        digest: generateDigest(content),
+                        body: content,
+                        rendered: {
+                            html: renderedContent.code,
+                            metadata: {
+                                headings: renderedContent.metadata.headings,
+                            },
+                        },
+                        data: {
+                            name: pipeline.name,
+                        },
+                    });
+                });
+            });
+        });
+    },
+};

sites/main-site/src/pages/blog/[...blogpost].astro

@@ -1,6 +1,6 @@
 ---
 import MarkdownTocLayout from '@layouts/MarkdownTocLayout.astro';
-import { getCollection } from 'astro:content';
+import { getCollection, render } from 'astro:content';
 import SidebarStatsRow from '@components/sidebar/SidebarStatsRow.astro';
 import GitHubProfilePictureExtended from '@components/GitHubProfilePictureExtended.svelte';
 import { Image } from 'astro:assets';
@@ -10,6 +10,7 @@ const images = import.meta.glob('../../assets/**');
 
 export async function getStaticPaths() {
     const blogPosts = await getCollection('blog');
+    console.log(blogPosts);
     return blogPosts.map((post) => ({
         params: { blogpost: post.slug },
         props: { post, blogPosts },
@@ -50,7 +51,7 @@ const sections: SidebarEntry[] = years.map((year) => {
             }),
     };
 });
-const { headings, Content } = await post.render();
+const { headings, Content } = await render(post);
 const title = post.data.title;
 const subtitle = post.data.subtitle;
 const md_github_url = 'https://github.com/nf-core/website/blob/main/sites/main-site/src/content/blog/' + post.id;