From 461671aa9d3ba640ccbef07599e60de5a8462d05 Mon Sep 17 00:00:00 2001 From: Marcos Navarro Date: Tue, 26 Nov 2024 01:07:19 +0900 Subject: [PATCH 1/3] Improve Quarto projects inspection by picking up pre and post scripts to the configuration and allowing to use quarto yml files as entrypoints --- internal/inspect/detectors/all.go | 6 +- internal/inspect/detectors/quarto.go | 264 ++++++++++++------ internal/inspect/detectors/quarto_test.go | 53 +++- .../quarto-website-via-yaml/_quarto.yml | 21 ++ .../quarto-website-via-yaml/about.qmd | 21 ++ .../testdata/quarto-website-via-yaml/all.qmd | 11 + .../quarto-website-via-yaml/finally.py | 1 + .../quarto-website-via-yaml/index.qmd | 13 + .../quarto-website-via-yaml/inspect.json | 53 ++++ .../quarto-website-via-yaml/prepare.py | 202 ++++++++++++++ .../quarto-website-via-yaml/requirements.txt | 10 + 11 files changed, 563 insertions(+), 92 deletions(-) create mode 100644 internal/inspect/detectors/testdata/quarto-website-via-yaml/_quarto.yml create mode 100644 internal/inspect/detectors/testdata/quarto-website-via-yaml/about.qmd create mode 100644 internal/inspect/detectors/testdata/quarto-website-via-yaml/all.qmd create mode 100644 internal/inspect/detectors/testdata/quarto-website-via-yaml/finally.py create mode 100644 internal/inspect/detectors/testdata/quarto-website-via-yaml/index.qmd create mode 100644 internal/inspect/detectors/testdata/quarto-website-via-yaml/inspect.json create mode 100644 internal/inspect/detectors/testdata/quarto-website-via-yaml/prepare.py create mode 100644 internal/inspect/detectors/testdata/quarto-website-via-yaml/requirements.txt diff --git a/internal/inspect/detectors/all.go b/internal/inspect/detectors/all.go index 8461f1679..7520c2bcd 100644 --- a/internal/inspect/detectors/all.go +++ b/internal/inspect/detectors/all.go @@ -13,11 +13,13 @@ import ( ) type ContentTypeDetector struct { + log logging.Logger detectors []ContentTypeInferer } func NewContentTypeDetector(log logging.Logger) *ContentTypeDetector { return &ContentTypeDetector{ + log: log, detectors: []ContentTypeInferer{ // The order here is important, since the first // ContentTypeInferer to return a non-nil @@ -27,7 +29,7 @@ func NewContentTypeDetector(log logging.Logger) *ContentTypeDetector { NewPlumberDetector(), NewRMarkdownDetector(log), NewNotebookDetector(), - NewQuartoDetector(), + NewQuartoDetector(log), NewRShinyDetector(), NewPyShinyDetector(), NewFastAPIDetector(), @@ -104,5 +106,3 @@ func (t *ContentTypeDetector) InferType(base util.AbsolutePath, entrypoint util. slices.SortFunc(allConfigs, compareConfigs) return allConfigs, nil } - -var _ ContentTypeInferer = &ContentTypeDetector{} diff --git a/internal/inspect/detectors/quarto.go b/internal/inspect/detectors/quarto.go index 6a711e622..1369d9580 100644 --- a/internal/inspect/detectors/quarto.go +++ b/internal/inspect/detectors/quarto.go @@ -22,11 +22,11 @@ type QuartoDetector struct { log logging.Logger } -func NewQuartoDetector() *QuartoDetector { +func NewQuartoDetector(log logging.Logger) *QuartoDetector { return &QuartoDetector{ inferenceHelper: defaultInferenceHelper{}, executor: executor.NewExecutor(), - log: logging.New(), + log: log, } } @@ -36,6 +36,18 @@ type quartoMetadata struct { Server any `json:"server"` } +type quartoProjectConfig struct { + Project struct { + Title string `json:"title"` + PreRender []string `json:"pre-render"` + PostRender []string `json:"post-render"` + OutputDir string `json:"output-dir"` + } `json:"project"` + Website struct { + Title string `json:"title"` + } `json:"website"` +} + type quartoInspectOutput struct { // Only the fields we use are included; the rest // are discarded by the JSON decoder. @@ -43,18 +55,8 @@ type quartoInspectOutput struct { Version string `json:"version"` } `json:"quarto"` Project struct { - Config struct { - Project struct { - Title string `json:"title"` - PreRender []string `json:"pre-render"` - PostRender []string `json:"post-render"` - OutputDir string `json:"output-dir"` - } `json:"project"` - Website struct { - Title string `json:"title"` - } `json:"website"` - } `json:"config"` - Files struct { + Config quartoProjectConfig `json:"config"` + Files struct { Input []string `json:"input"` } `json:"files"` } `json:"project"` @@ -81,6 +83,9 @@ type quartoInspectOutput struct { // For single quarto docs without _quarto.yml, // there is no project section in the output. FileInformation map[string]any `json:"fileInformation"` + + // For directory inspect (commonly due to _quarto.yml picked up as entrypoint) + Config quartoProjectConfig `json:"config"` } func (d *QuartoDetector) quartoInspect(path util.AbsolutePath) (*quartoInspectOutput, error) { @@ -114,6 +119,27 @@ func getInputFiles(inspectOutput *quartoInspectOutput) []string { return []string{} } +func getPrePostRenderFiles(inspectOutput *quartoInspectOutput) []string { + filenames := []string{} + preRender := inspectOutput.Config.Project.PreRender + preRenderAlt := inspectOutput.Project.Config.Project.PreRender + postRender := inspectOutput.Config.Project.PostRender + postRenderAlt := inspectOutput.Project.Config.Project.PostRender + if preRender != nil { + filenames = append(filenames, preRender...) + } + if preRenderAlt != nil { + filenames = append(filenames, preRenderAlt...) + } + if postRender != nil { + filenames = append(filenames, postRender...) + } + if postRenderAlt != nil { + filenames = append(filenames, postRenderAlt...) + } + return filenames +} + func (d *QuartoDetector) needsPython(inspectOutput *quartoInspectOutput) bool { if inspectOutput == nil { return false @@ -164,6 +190,13 @@ func (d *QuartoDetector) getTitle(inspectOutput *quartoInspectOutput, entrypoint if isValidTitle(inspectOutput.Formats.RevealJS.Metadata.Title) { return inspectOutput.Formats.RevealJS.Metadata.Title } + // Config data can exist at root level or within an additional Project object + if isValidTitle(inspectOutput.Config.Website.Title) { + return inspectOutput.Config.Website.Title + } + if isValidTitle(inspectOutput.Config.Project.Title) { + return inspectOutput.Config.Project.Title + } if isValidTitle(inspectOutput.Project.Config.Website.Title) { return inspectOutput.Project.Config.Website.Title } @@ -201,49 +234,39 @@ func isQuartoShiny(metadata *quartoMetadata) bool { return false } -func (d *QuartoDetector) InferType(base util.AbsolutePath, entrypoint util.RelativePath) ([]*config.Config, error) { - if entrypoint.String() != "" { - // Optimization: skip inspection if there's a specified entrypoint - // and it's not one of ours. - if !slices.Contains(quartoSuffixes, entrypoint.Ext()) { - return nil, nil - } +func (d *QuartoDetector) configFromFileInspect(base util.AbsolutePath, entrypointPath util.AbsolutePath) (*config.Config, error) { + inspectOutput, err := d.quartoInspect(entrypointPath) + if err != nil { + // Maybe this isn't really a quarto project, or maybe the user doesn't have quarto. + // We log this error and continue checking the other files. + d.log.Warn("quarto inspect failed", "file", entrypointPath.String(), "error", err) + return nil, nil } - var configs []*config.Config - entrypointPaths, err := d.findEntrypoints(base) + + relEntrypoint, err := entrypointPath.Rel(base) if err != nil { return nil, err } - for _, entrypointPath := range entrypointPaths { - relEntrypoint, err := entrypointPath.Rel(base) - if err != nil { - return nil, err - } - if entrypoint.String() != "" && relEntrypoint != entrypoint { - // Only inspect the specified file - continue - } - inspectOutput, err := d.quartoInspect(entrypointPath) - if err != nil { - // Maybe this isn't really a quarto project, or maybe the user doesn't have quarto. - // We log this error and continue checking the other files. - d.log.Warn("quarto inspect failed", "file", entrypointPath.String(), "error", err) - continue - } - cfg := config.New() - cfg.Entrypoint = relEntrypoint.String() - cfg.Title = d.getTitle(inspectOutput, relEntrypoint.String()) + cfg := config.New() + cfg.Entrypoint = relEntrypoint.String() + cfg.Title = d.getTitle(inspectOutput, relEntrypoint.String()) - if isQuartoShiny(&inspectOutput.Formats.HTML.Metadata) || - isQuartoShiny(&inspectOutput.Formats.RevealJS.Metadata) { - cfg.Type = config.ContentTypeQuartoShiny - } else { - cfg.Type = config.ContentTypeQuarto - } + if isQuartoShiny(&inspectOutput.Formats.HTML.Metadata) || + isQuartoShiny(&inspectOutput.Formats.RevealJS.Metadata) { + cfg.Type = config.ContentTypeQuartoShiny + } else { + cfg.Type = config.ContentTypeQuarto + } - var needR, needPython bool + var needR, needPython bool + isDir, err := entrypointPath.IsDir() + if err != nil { + return nil, err + } + + if !isDir { if entrypointPath.HasSuffix(".ipynb") { needPython = true } else { @@ -254,54 +277,119 @@ func (d *QuartoDetector) InferType(base util.AbsolutePath, entrypoint util.Relat } needR, needPython = pydeps.DetectMarkdownLanguagesInContent(content) } - engines := inspectOutput.Engines - if needPython || d.needsPython(inspectOutput) { - // Indicate that Python inspection is needed. - cfg.Python = &config.Python{} - if !slices.Contains(engines, "jupyter") { - engines = append(engines, "jupyter") - } + } + + engines := inspectOutput.Engines + if needPython || d.needsPython(inspectOutput) { + // Indicate that Python inspection is needed. + cfg.Python = &config.Python{} + if !slices.Contains(engines, "jupyter") { + engines = append(engines, "jupyter") } - if needR || d.needsR(inspectOutput) { - // Indicate that R inspection is needed. - cfg.R = &config.R{} - if !slices.Contains(engines, "knitr") { - engines = append(engines, "knitr") - } + } + if needR || d.needsR(inspectOutput) { + // Indicate that R inspection is needed. + cfg.R = &config.R{} + if !slices.Contains(engines, "knitr") { + engines = append(engines, "knitr") } - slices.Sort(engines) + } + slices.Sort(engines) - cfg.Quarto = &config.Quarto{ - Version: inspectOutput.Quarto.Version, - Engines: engines, - } + cfg.Quarto = &config.Quarto{ + Version: inspectOutput.Quarto.Version, + Engines: engines, + } - // Only include the entrypoint and its associated files. - inputFiles := getInputFiles(inspectOutput) - for _, inputFile := range inputFiles { - var relPath string - if filepath.IsAbs(inputFile) { - relPath, err = filepath.Rel(base.String(), inputFile) - if err != nil { - return nil, err - } - } else { - relPath = inputFile - } - cfg.Files = append(cfg.Files, fmt.Sprint("/", relPath)) - } - extraFiles := []string{"_quarto.yml", "_metadata.yaml"} - for _, filename := range extraFiles { - path := base.Join(filename) - exists, err := path.Exists() + // Include the entrypoint, its associated files and pre-post render scripts + filesToInclude := getInputFiles(inspectOutput) + prepostscripts := getPrePostRenderFiles(inspectOutput) + filesToInclude = append(filesToInclude, prepostscripts...) + for _, inputFile := range filesToInclude { + var relPath string + if filepath.IsAbs(inputFile) { + relPath, err = filepath.Rel(base.String(), inputFile) if err != nil { return nil, err } - if exists { - cfg.Files = append(cfg.Files, fmt.Sprint("/", filename)) + } else { + relPath = inputFile + } + cfg.Files = append(cfg.Files, fmt.Sprint("/", relPath)) + } + + extraFiles := []string{"_quarto.yml", "_quarto.yaml", "_metadata.yml", "_metadata.yaml"} + for _, filename := range extraFiles { + path := base.Join(filename) + exists, err := path.Exists() + if err != nil { + return nil, err + } + + if exists { + cfg.Files = append(cfg.Files, fmt.Sprint("/", filename)) + // Update entrypoint to keep the original _quarto.* file + if cfg.Entrypoint == "." { + cfg.Entrypoint = filename } } - configs = append(configs, cfg) + } + return cfg, nil +} + +func (d *QuartoDetector) isQuartoYaml(entrypointBase string) bool { + return slices.Contains([]string{"_quarto.yml", "_quarto.yaml"}, entrypointBase) +} + +func (d *QuartoDetector) InferType(base util.AbsolutePath, entrypoint util.RelativePath) ([]*config.Config, error) { + // When the choosen entrypoint is _quarto.yml, "quarto inspect" command does not handle it well + // but an inspection to the base directory will bring what the user expects in this case. + if d.isQuartoYaml(entrypoint.Base()) { + d.log.Debug("A _quarto.yml file was picked as entrypoint", "inspect_path", base.String()) + + cfg, err := d.configFromFileInspect(base, base) + if err != nil { + return nil, err + } + + if cfg != nil { + return []*config.Config{cfg}, nil + } + } + + if entrypoint.String() != "" { + // Optimization: skip inspection if there's a specified entrypoint + // and it's not one of ours. + if !slices.Contains(quartoSuffixes, entrypoint.Ext()) { + d.log.Debug("Picked entrypoint does not match Quarto file extensions, skipping", "entrypoint", entrypoint.String()) + return nil, nil + } + } + + var configs []*config.Config + entrypointPaths, err := d.findEntrypoints(base) + if err != nil { + return nil, err + } + + for _, entrypointPath := range entrypointPaths { + relEntrypoint, err := entrypointPath.Rel(base) + if err != nil { + return nil, err + } + if entrypoint.String() != "" && relEntrypoint != entrypoint { + // Only inspect the specified file + continue + } + + cfg, err := d.configFromFileInspect(base, entrypointPath) + if err != nil { + return nil, err + } + + if cfg != nil { + configs = append(configs, cfg) + } } return configs, nil } diff --git a/internal/inspect/detectors/quarto_test.go b/internal/inspect/detectors/quarto_test.go index 95697b505..54748833a 100644 --- a/internal/inspect/detectors/quarto_test.go +++ b/internal/inspect/detectors/quarto_test.go @@ -12,6 +12,7 @@ import ( "github.com/posit-dev/publisher/internal/config" "github.com/posit-dev/publisher/internal/executor/executortest" + "github.com/posit-dev/publisher/internal/logging" "github.com/posit-dev/publisher/internal/schema" "github.com/posit-dev/publisher/internal/util" "github.com/posit-dev/publisher/internal/util/utiltest" @@ -33,7 +34,7 @@ func (s *QuartoDetectorSuite) runInferType(testName string) []*config.Config { base := realCwd.Join("testdata", testName) - detector := NewQuartoDetector() + detector := NewQuartoDetector(logging.New()) executor := executortest.NewMockExecutor() detector.executor = executor @@ -249,6 +250,56 @@ func (s *QuartoDetectorSuite) TestInferTypeQuartoWebsite() { }, configs[1]) } +func (s *QuartoDetectorSuite) TestInferTypeQuartoWebsite_viaQuartoYml() { + if runtime.GOOS == "windows" { + s.T().Skip("This test does not run on Windows") + } + // configs := s.runInferType("quarto-website-none") + realCwd, err := util.Getwd(nil) + s.NoError(err) + + base := realCwd.Join("testdata", "quarto-website-via-yaml") + + detector := NewQuartoDetector(logging.New()) + executor := executortest.NewMockExecutor() + detector.executor = executor + + dirOutputPath := base.Join("inspect.json") + exists, err := dirOutputPath.Exists() + s.NoError(err) + s.True(exists) + + // Replace the $DIR placeholder in the file with + // the correct path (json-escaped) + placeholder := []byte("$DIR") + baseDir, err := json.Marshal(base.Dir().String()) + s.NoError(err) + baseDir = baseDir[1 : len(baseDir)-1] + + dirOutput, err := dirOutputPath.ReadFile() + s.NoError(err) + dirOutput = bytes.ReplaceAll(dirOutput, placeholder, baseDir) + executor.On("RunCommand", "quarto", []string{"inspect", base.String()}, mock.Anything, mock.Anything).Return(dirOutput, nil, nil) + + configs, err := detector.InferType(base, util.NewRelativePath("_quarto.yml", nil)) + s.Nil(err) + + s.Len(configs, 1) + s.Equal(&config.Config{ + Schema: schema.ConfigSchemaURL, + Type: config.ContentTypeQuarto, + Entrypoint: "_quarto.yml", + Title: "Content Dashboard", + Validate: true, + Files: []string{"/all.qmd", "/index.qmd", "/about.qmd", "/prepare.py", "/finally.py", "/_quarto.yml"}, + Quarto: &config.Quarto{ + Version: "1.4.553", + Engines: []string{"jupyter", "markdown"}, + }, + Python: &config.Python{}, + }, configs[0]) +} + func (s *QuartoDetectorSuite) TestInferTypeRMarkdownDoc() { if runtime.GOOS == "windows" { s.T().Skip("This test does not run on Windows") diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/_quarto.yml b/internal/inspect/detectors/testdata/quarto-website-via-yaml/_quarto.yml new file mode 100644 index 000000000..a91a8c399 --- /dev/null +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/_quarto.yml @@ -0,0 +1,21 @@ +project: + type: website + pre-render: prepare.py + +website: + title: Content Dashboard + navbar: + left: + - text: "Featured" + href: "index.qmd" + - text: "All" + href: "all.qmd" + - text: "About" + href: "about.qmd" + +format: + html: default + +theme: + light: flatly + dark: darkly diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/about.qmd b/internal/inspect/detectors/testdata/quarto-website-via-yaml/about.qmd new file mode 100644 index 000000000..c2d4123d8 --- /dev/null +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/about.qmd @@ -0,0 +1,21 @@ +--- +title: About +--- + +This [Quarto Website](https://quarto.org/docs/websites/) shows how the [Posit +Connect Server API](https://docs.posit.co/connect/api/) can combine with +[Quarto Listings](https://quarto.org/docs/websites/website-listings.html) to +create dashboards that highlight your content. + +```{python} +#| echo: false +#| output: asis + +import yaml +with open("featured.yaml", "r") as f: + featured = len(yaml.safe_load(f)) +with open("all.yaml", "r") as f: + listing = len(yaml.safe_load(f)) + +print(f"This dashboard presents {featured} featured content items with {listing} total items.") +``` diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/all.qmd b/internal/inspect/detectors/testdata/quarto-website-via-yaml/all.qmd new file mode 100644 index 000000000..866d67d90 --- /dev/null +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/all.qmd @@ -0,0 +1,11 @@ +--- +title: All Content +listing: + contents: all.yaml + type: table + fields: + - date + - title + - author + max-items: 200 +--- diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/finally.py b/internal/inspect/detectors/testdata/quarto-website-via-yaml/finally.py new file mode 100644 index 000000000..47b36b861 --- /dev/null +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/finally.py @@ -0,0 +1 @@ +print("finished") diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/index.qmd b/internal/inspect/detectors/testdata/quarto-website-via-yaml/index.qmd new file mode 100644 index 000000000..223ce5f9c --- /dev/null +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/index.qmd @@ -0,0 +1,13 @@ +--- +title: Featured Content +listing: + contents: featured.yaml + type: grid + categories: true + fields: + - image + - date + - title + - author + - categories +--- diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/inspect.json b/internal/inspect/detectors/testdata/quarto-website-via-yaml/inspect.json new file mode 100644 index 000000000..4638304fd --- /dev/null +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/inspect.json @@ -0,0 +1,53 @@ +{ + "quarto": { + "version": "1.4.553" + }, + "dir": "$DIR/quarto-website-via-yaml", + "engines": ["markdown", "jupyter"], + "config": { + "project": { + "type": "website", + "pre-render": ["prepare.py"], + "post-render": ["finally.py"], + "lib-dir": "site_libs", + "output-dir": "_site" + }, + "website": { + "title": "Content Dashboard", + "navbar": { + "left": [ + { + "text": "Featured", + "href": "index.qmd" + }, + { + "text": "All", + "href": "all.qmd" + }, + { + "text": "About", + "href": "about.qmd" + } + ] + } + }, + "format": { + "html": "default" + }, + "theme": { + "light": "flatly", + "dark": "darkly" + }, + "language": {} + }, + "files": { + "input": [ + "$DIR/quarto-website-via-yaml/all.qmd", + "$DIR/quarto-website-via-yaml/index.qmd", + "$DIR/quarto-website-via-yaml/about.qmd" + ], + "resources": [], + "config": ["$DIR/quarto-website-via-yaml/_quarto.yml"], + "configResources": [] + } +} diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/prepare.py b/internal/inspect/detectors/testdata/quarto-website-via-yaml/prepare.py new file mode 100644 index 000000000..7d996b9d6 --- /dev/null +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/prepare.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 + +# +# Python script that uses the Posit Connect Server API to enumerate content to +# a Posit Connect server. +# +# Environment variables: +# +# * CONNECT_SERVER - The URL for your Posit Connect installation. +# +# * CONNECT_API_KEY - A Posit Connect API key. +# +# * FEATURED_TAGS - A comma-separated list of tag names. Content using any of +# these tags is featured. +# +# Example configuration: +# +# CONNECT_SERVER="https://connect.company.com/" +# CONNECT_API_KEY="NM6ZI4vluEHsyg5ViV3zK2bhBGqjiayA" +# FEATURED_TAGS="showcase" +# +# Produces two YAML files used by the Quarto website: +# +# * featured.yaml - YAML file listing content to feature on the landing page. +# +# * all.yaml - YAML file listing all available content. +# + +import os +import sys +import requests +import yaml + +connectServer = os.getenv("CONNECT_SERVER") +if not connectServer: + print("ERROR: Environment variable CONNECT_SERVER must be defined.") + sys.exit(1) + +connectAPIKey = os.getenv("CONNECT_API_KEY") +if not connectAPIKey: + print("ERROR: Environment variable CONNECT_API_KEY must be defined.") + sys.exit(1) + +featuredTags = os.getenv("FEATURED_TAGS") +if featuredTags is not None: + featuredTags = [tag.strip() for tag in featuredTags.split(",")] + featuredTags = [tag for tag in featuredTags if tag] + +# Ensure that connectServer has a trailing slash. +if connectServer[-1] != "/": + connectServer = connectServer + "/" + + +def text_escape(text): + """ + Helper to escape some characters that present problems if they appear + in the Quarto listing YAML text. + + See: https://github.com/quarto-dev/quarto-cli/issues/6745 + """ + return text.replace("&", "&").replace("@", "@").replace("$", "$") + + +def default_icon(item): + """ + Returns an icon appropriate for the content item. + """ + app_mode = item["app_mode"] + content_category = item["content_category"] + if app_mode == "api": + return "api.svg" + elif app_mode == "shiny": + return "app.svg" + elif app_mode == "rmd-shiny": + return "doc.svg" + elif app_mode == "quarto-shiny": + return "doc.svg" + elif app_mode == "rmd-static": + return "doc.svg" + elif app_mode == "quarto-static": + return "doc.svg" + elif app_mode == "tensorflow-saved-model": + return "model.svg" + elif app_mode == "python-api": + return "api.svg" + elif app_mode == "python-dash": + return "app.svg" + elif app_mode == "python-gradio": + return "app.svg" + elif app_mode == "python-streamlit": + return "app.svg" + elif app_mode == "python-bokeh": + return "app.svg" + elif app_mode == "python-shiny": + return "app.svg" + elif app_mode == "static": + if content_category == "plot": + return "plot.svg" + elif content_category == "pin": + return "pin.svg" + + return "doc.svg" + + +def content_image(item): + """ + Returns the path to an image for this content item. By default, this + uses an icon associated with the type of content. + """ + return f"icons/{default_icon(item)}" + + +def listing_item_from_content(item): + """ + Helper to transform a content item returned by the Posit Connect + Server API into an entry that is compatible with the Quarto document + listing YAML. + """ + title = item["title"] + if not title: + title = item["name"] + + owner = item["owner"] + + record = { + "guid": item["guid"], + "app_mode": item["app_mode"], + "content_category": item["content_category"], + "title": text_escape(title), + "author": text_escape(f'{owner["first_name"]} {owner["last_name"]}'), + "date": item["last_deployed_time"], + # href? filename? path? + "path": item["content_url"], + "image": content_image(item), + } + + if item["description"]: + record["description"] = text_escape(item["description"]) + + tags = item.get("tags") + if tags: + record["categories"] = [tag["name"] for tag in tags] + + return record + + +def listing_items_from_content(items): + """ + Helper to transform a list of content items returned by the Posit + Connect Server API into a collection that is compatible with the Quarto + document listing YAML. + """ + return [listing_item_from_content(item) for item in items] + + +def filter_listing(listing, tags): + """ + helper that returns listing entries that reference any tag. + """ + tagset = set(tags) + return [item for item in listing if tagset.intersection(item.get("categories", []))] + + +def write_yaml(filename, listing): + """ + Helper that writes a YAML file from a set of listing records that are + compatible with the Quarto document listing format. + """ + with open(filename, "w") as f: + yaml.safe_dump(listing, f) + + +# List all content. +# https://docs.posit.co/connect/api/#get-/v1/content +r = requests.get( + f"{connectServer}__api__/v1/content", + params={ + "include": "tags,owner", + }, + headers={ + "Authorization": "Key " + connectAPIKey, + }, +) +r.raise_for_status() +payload = r.json() + +# payload = payload[:100] +listing = listing_items_from_content(payload) +print(f"Fetched {len(listing)} items.") +write_yaml("all.yaml", listing) + +# When we have featured tags, show only content that uses those tags. +# Otherwise, show the 20 most recent items. +featured = [] +if featuredTags: + print("Featuring items having tags: %s" % ", ".join(featuredTags)) + featured = filter_listing(listing, featuredTags) + print(f"Filtering by tags selected {len(featured)} items.") +if not featured: + print("No featured tags, or no matching items. Featuring 20 most recent items") + featured = listing[:20] +write_yaml("featured.yaml", featured) diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/requirements.txt b/internal/inspect/detectors/testdata/quarto-website-via-yaml/requirements.txt new file mode 100644 index 000000000..aae377b3c --- /dev/null +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/requirements.txt @@ -0,0 +1,10 @@ +# jupyter +notebook==7.2.2 +qtconsole==5.5.1 +jupyter-console==6.6.3 +nbconvert==7.16.1 +ipykernel==6.29.3 +ipywidgets==8.1.2 + +requests==2.32.2 +pyyaml==6.0.1 From f10b86dd817124d5734c0e7c87d26d9e5d913b01 Mon Sep 17 00:00:00 2001 From: Marcos Navarro Date: Wed, 27 Nov 2024 22:45:19 +0900 Subject: [PATCH 2/3] A few more improvements on Quarto projects files inspection --- internal/inspect/detectors/quarto.go | 71 +++++++++++++++++-- internal/inspect/detectors/quarto_test.go | 12 +++- .../quarto-website-via-yaml/_brand.yml | 17 +++++ .../quarto-website-via-yaml/inspect.json | 50 ++++++++++++- 4 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 internal/inspect/detectors/testdata/quarto-website-via-yaml/_brand.yml diff --git a/internal/inspect/detectors/quarto.go b/internal/inspect/detectors/quarto.go index 1369d9580..f267adf74 100644 --- a/internal/inspect/detectors/quarto.go +++ b/internal/inspect/detectors/quarto.go @@ -16,6 +16,18 @@ import ( "github.com/posit-dev/publisher/internal/util" ) +// These are files that behave in a special way for Quarto +// and are not included within project inspection output, +// if exists must be included in deployment files. +var specialYmlFiles = []string{ + "_quarto.yml", + "_quarto.yaml", + "_metadata.yml", + "_metadata.yaml", + "_brand.yml", + "_brand.yaml", +} + type QuartoDetector struct { inferenceHelper executor executor.Executor @@ -62,7 +74,8 @@ type quartoInspectOutput struct { } `json:"project"` Engines []string `json:"engines"` Files struct { - Input []string `json:"input"` + Input []string `json:"input"` + ConfigResources []string `json:"configResources"` } `json:"files"` // For single quarto docs without _quarto.yml Formats struct { @@ -82,7 +95,11 @@ type quartoInspectOutput struct { // For single quarto docs without _quarto.yml, // there is no project section in the output. - FileInformation map[string]any `json:"fileInformation"` + FileInformation map[string]struct { + Metadata struct { + ResourceFiles []string `json:"resource_files"` + } `json:"metadata"` + } `json:"fileInformation"` // For directory inspect (commonly due to _quarto.yml picked up as entrypoint) Config quartoProjectConfig `json:"config"` @@ -119,6 +136,48 @@ func getInputFiles(inspectOutput *quartoInspectOutput) []string { return []string{} } +func getConfigResources(inspectOutput *quartoInspectOutput) []string { + if inspectOutput.Files.ConfigResources != nil { + return inspectOutput.Files.ConfigResources + } + return []string{} +} + +func uniqueFileResources(target []string, resourceFiles []string) []string { + resourcesFound := []string{} + for _, fileResource := range resourceFiles { + // Prevent duplicated files + if slices.Contains(target, fileResource) { + continue + } + + // Prevent special YML files here (those are handled later) + if slices.Contains(specialYmlFiles, fileResource) { + continue + } + + resourcesFound = append(resourcesFound, fileResource) + } + return resourcesFound +} + +func getFileInfoResources(inspectOutput *quartoInspectOutput) []string { + if inspectOutput.FileInformation == nil { + return []string{} + } + + filesResources := []string{} + for _, fileInfo := range inspectOutput.FileInformation { + if fileInfo.Metadata.ResourceFiles != nil { + filesResources = append( + filesResources, + uniqueFileResources(filesResources, fileInfo.Metadata.ResourceFiles)...) + } + } + + return filesResources +} + func getPrePostRenderFiles(inspectOutput *quartoInspectOutput) []string { filenames := []string{} preRender := inspectOutput.Config.Project.PreRender @@ -303,8 +362,9 @@ func (d *QuartoDetector) configFromFileInspect(base util.AbsolutePath, entrypoin // Include the entrypoint, its associated files and pre-post render scripts filesToInclude := getInputFiles(inspectOutput) - prepostscripts := getPrePostRenderFiles(inspectOutput) - filesToInclude = append(filesToInclude, prepostscripts...) + filesToInclude = append(filesToInclude, getConfigResources(inspectOutput)...) + filesToInclude = append(filesToInclude, getFileInfoResources(inspectOutput)...) + filesToInclude = append(filesToInclude, getPrePostRenderFiles(inspectOutput)...) for _, inputFile := range filesToInclude { var relPath string if filepath.IsAbs(inputFile) { @@ -318,8 +378,7 @@ func (d *QuartoDetector) configFromFileInspect(base util.AbsolutePath, entrypoin cfg.Files = append(cfg.Files, fmt.Sprint("/", relPath)) } - extraFiles := []string{"_quarto.yml", "_quarto.yaml", "_metadata.yml", "_metadata.yaml"} - for _, filename := range extraFiles { + for _, filename := range specialYmlFiles { path := base.Join(filename) exists, err := path.Exists() if err != nil { diff --git a/internal/inspect/detectors/quarto_test.go b/internal/inspect/detectors/quarto_test.go index 54748833a..5b0c343dd 100644 --- a/internal/inspect/detectors/quarto_test.go +++ b/internal/inspect/detectors/quarto_test.go @@ -291,7 +291,17 @@ func (s *QuartoDetectorSuite) TestInferTypeQuartoWebsite_viaQuartoYml() { Entrypoint: "_quarto.yml", Title: "Content Dashboard", Validate: true, - Files: []string{"/all.qmd", "/index.qmd", "/about.qmd", "/prepare.py", "/finally.py", "/_quarto.yml"}, + Files: []string{ + "/all.qmd", + "/index.qmd", + "/about.qmd", + "/bibliography.bib", + "/palmer-penguins.csv", + "/prepare.py", + "/finally.py", + "/_quarto.yml", + "/_brand.yml", + }, Quarto: &config.Quarto{ Version: "1.4.553", Engines: []string{"jupyter", "markdown"}, diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/_brand.yml b/internal/inspect/detectors/testdata/quarto-website-via-yaml/_brand.yml new file mode 100644 index 000000000..7d677331a --- /dev/null +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/_brand.yml @@ -0,0 +1,17 @@ +color: + palette: + dark-grey: "#222222" + blue: "#ddeaf1" + background: blue + foreground: dark-grey + primary: black + +logo: + medium: logo.png + +typography: + fonts: + - family: Jura + source: google + base: Jura + headings: Jura diff --git a/internal/inspect/detectors/testdata/quarto-website-via-yaml/inspect.json b/internal/inspect/detectors/testdata/quarto-website-via-yaml/inspect.json index 4638304fd..52728fcb4 100644 --- a/internal/inspect/detectors/testdata/quarto-website-via-yaml/inspect.json +++ b/internal/inspect/detectors/testdata/quarto-website-via-yaml/inspect.json @@ -48,6 +48,54 @@ ], "resources": [], "config": ["$DIR/quarto-website-via-yaml/_quarto.yml"], - "configResources": [] + "configResources": ["bibliography.bib"] + }, + "fileInformation": { + "$DIR/quarto-website-via-yaml/all.qmd": { + "includeMap": [], + "codeCells": [], + "metadata": { + "title": "All Content", + "listing": { + "contents": "all.yaml", + "type": "table", + "fields": ["date", "title", "author"], + "max-items": 200 + } + } + }, + "$DIR/quarto-website-via-yaml/index.qmd": { + "includeMap": [], + "codeCells": [], + "metadata": { + "title": "Featured Content", + "resource_files": ["_quarto.yml", "palmer-penguins.csv"], + "listing": { + "contents": "featured.yaml", + "type": "grid", + "categories": true, + "fields": ["image", "date", "title", "author", "categories"] + } + } + }, + "$DIR/quarto-website-via-yaml/about.qmd": { + "includeMap": [], + "codeCells": [ + { + "start": 9, + "end": 20, + "file": "$DIR/quarto-website-via-yaml/about.qmd", + "source": "\nimport yaml\nwith open(\"featured.yaml\", \"r\") as f:\n featured = len(yaml.safe_load(f))\nwith open(\"all.yaml\", \"r\") as f:\n listing = len(yaml.safe_load(f))\n\nprint(f\"This dashboard presents {featured} featured content items with {listing} total items.\")\n", + "language": "python", + "metadata": { + "echo": false, + "output": "asis" + } + } + ], + "metadata": { + "title": "About" + } + } } } From 21cdcd34a7756537671a9a4858512e28191f4580 Mon Sep 17 00:00:00 2001 From: Marcos Navarro Date: Tue, 3 Dec 2024 19:06:32 +0900 Subject: [PATCH 3/3] Include quarto config resources on individual file inspection too --- internal/inspect/detectors/quarto.go | 21 +++++++++++++-------- internal/inspect/detectors/quarto_test.go | 7 ++++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/internal/inspect/detectors/quarto.go b/internal/inspect/detectors/quarto.go index f267adf74..439442bad 100644 --- a/internal/inspect/detectors/quarto.go +++ b/internal/inspect/detectors/quarto.go @@ -60,6 +60,11 @@ type quartoProjectConfig struct { } `json:"website"` } +type quartoFilesData struct { + Input []string `json:"input"` + ConfigResources []string `json:"configResources"` +} + type quartoInspectOutput struct { // Only the fields we use are included; the rest // are discarded by the JSON decoder. @@ -68,15 +73,10 @@ type quartoInspectOutput struct { } `json:"quarto"` Project struct { Config quartoProjectConfig `json:"config"` - Files struct { - Input []string `json:"input"` - } `json:"files"` + Files quartoFilesData `json:"files"` } `json:"project"` - Engines []string `json:"engines"` - Files struct { - Input []string `json:"input"` - ConfigResources []string `json:"configResources"` - } `json:"files"` + Engines []string `json:"engines"` + Files quartoFilesData `json:"files"` // For single quarto docs without _quarto.yml Formats struct { HTML struct { @@ -137,6 +137,11 @@ func getInputFiles(inspectOutput *quartoInspectOutput) []string { } func getConfigResources(inspectOutput *quartoInspectOutput) []string { + // When inspection is done on a specific file (e.g: picking index.qmd as entrypoint) + if inspectOutput.Project.Files.ConfigResources != nil { + return inspectOutput.Project.Files.ConfigResources + } + // When inspection is done on directory (picking _quarto.yml as entrypoint) if inspectOutput.Files.ConfigResources != nil { return inspectOutput.Files.ConfigResources } diff --git a/internal/inspect/detectors/quarto_test.go b/internal/inspect/detectors/quarto_test.go index 5b0c343dd..75731f0ac 100644 --- a/internal/inspect/detectors/quarto_test.go +++ b/internal/inspect/detectors/quarto_test.go @@ -242,7 +242,12 @@ func (s *QuartoDetectorSuite) TestInferTypeQuartoWebsite() { Entrypoint: "index.qmd", Title: "quarto-website-none", Validate: true, - Files: []string{"/index.qmd", "/about.qmd", "/_quarto.yml"}, + Files: []string{ + "/index.qmd", + "/about.qmd", + "/styles.css", + "/_quarto.yml", + }, Quarto: &config.Quarto{ Version: "1.4.553", Engines: []string{"markdown"},