Big upstream changes from Project Forge

app/controller/clib/about.go

@@ -13,8 +13,7 @@ import (
 
 func About(rc *fasthttp.RequestCtx) {
 	controller.Act("about", rc, func(as *app.State, ps *cutil.PageState) (string, error) {
-		ps.Title = "About " + util.AppName
-		ps.Data = util.AppName + " v" + as.BuildInfo.Version
+		ps.SetTitleAndData("About "+util.AppName, util.AppName+" v"+as.BuildInfo.Version)
 		return controller.Render(rc, as, &views.About{}, ps, "about")
 	})
 }

app/controller/clib/admin.go

@@ -29,8 +29,7 @@ func Admin(rc *fasthttp.RequestCtx) {
 	}
 	controller.Act(key, rc, func(as *app.State, ps *cutil.PageState) (string, error) {
 		if len(path) == 0 {
-			ps.Title = "Administration"
-			ps.Data = "administration"
+			ps.SetTitleAndData("Administration", "administration")
 			return controller.Render(rc, as, &vadmin.Settings{Perms: user.GetPermissions()}, ps, "admin")
 		}
 		ps.DefaultNavIcon = "cog"
@@ -65,33 +64,27 @@ func Admin(rc *fasthttp.RequestCtx) {
 			}
 			return controller.FlashAndRedir(true, "wrote heap profile", "/admin", rc, ps)
 		case "logs":
-			ps.Title = "Recent Logs"
-			ps.Data = log.RecentLogs
+			ps.SetTitleAndData("Recent Logs", log.RecentLogs)
 			return controller.Render(rc, as, &vadmin.Logs{Logs: log.RecentLogs}, ps, "admin", "Recent Logs**folder")
 		case "memusage":
 			x := util.DebugMemStats()
 			ps.Data = x
 			return controller.Render(rc, as, &vadmin.MemUsage{Mem: x}, ps, "admin", "Memory Usage**desktop")
 		case "modules":
 			di := util.DebugBuildInfo().Deps
-			ps.Title = "Modules"
-			ps.Data = di
+			ps.SetTitleAndData("Modules", di)
 			return controller.Render(rc, as, &vadmin.Modules{Modules: di}, ps, "admin", "Modules**robot")
 		case "request":
-			ps.Title = "Request Debug"
-			ps.Data = cutil.RequestCtxToMap(rc, as, ps)
+			ps.SetTitleAndData("Request Debug", cutil.RequestCtxToMap(rc, as, ps))
 			return controller.Render(rc, as, &vadmin.Request{RC: rc}, ps, "admin", "Request**download")
 		case "routes":
-			ps.Title = "HTTP Routes"
-			ps.Data = AppRoutesList
+			ps.SetTitleAndData("HTTP Routes", AppRoutesList)
 			return controller.Render(rc, as, &vadmin.Routes{Routes: AppRoutesList}, ps, "admin", "Routes**folder")
 		case "session":
-			ps.Title = "Session Debug"
-			ps.Data = ps.Session
+			ps.SetTitleAndData("Session Debug", ps.Session)
 			return controller.Render(rc, as, &vadmin.Session{}, ps, "admin", "Session**play")
 		case "sitemap":
-			ps.Title = "Sitemap"
-			ps.Data = ps.Menu
+			ps.SetTitleAndData("Sitemap", ps.Menu)
 			return controller.Render(rc, as, &vadmin.Sitemap{}, ps, "admin", "Sitemap**graph")
 		case "sockets":
 			return socketRoute(rc, as, ps, path[1:]...)

app/controller/clib/profile.go

@@ -29,8 +29,7 @@ func ProfileSite(rc *fasthttp.RequestCtx) {
 }
 
 func profileAction(rc *fasthttp.RequestCtx, as *app.State, ps *cutil.PageState) (string, error) {
-	ps.Title = "Profile"
-	ps.Data = ps.Profile
+	ps.SetTitleAndData("Profile", ps.Profile)
 	thm := as.Themes.Get(ps.Profile.Theme, ps.Logger)
 
 	prvs, err := as.Auth.Providers(ps.Logger)

app/controller/clib/scripting.go

@@ -16,9 +16,8 @@ import (
 
 func ScriptingList(rc *fasthttp.RequestCtx) {
 	controller.Act("scripting.list", rc, func(as *app.State, ps *cutil.PageState) (string, error) {
-		ps.Title = "Scripting"
 		ret, sizes := as.Services.Script.ListScriptSizes(ps.Logger)
-		ps.Data = ret
+		ps.SetTitleAndData("Scripting", ret)
 		return controller.Render(rc, as, &vscripting.List{Scripts: ret, Sizes: sizes}, ps, "scripting")
 	})
 }
@@ -87,8 +86,7 @@ func ScriptingForm(rc *fasthttp.RequestCtx) {
 		if err != nil {
 			return "", err
 		}
-		ps.Title = "Edit [" + key + "]"
-		ps.Data = sc
+		ps.SetTitleAndData("Edit ["+key+"]", sc)
 		return controller.Render(rc, as, &vscripting.Form{Path: key, Content: sc}, ps, "scripting", key, "Edit")
 	})
 }

app/controller/clib/search.go

@@ -13,20 +13,21 @@ import (
 	"github.com/kyleu/npn/views/vsearch"
 )
 
+const searchKey = "search"
+
 func Search(rc *fasthttp.RequestCtx) {
-	controller.Act("search", rc, func(as *app.State, ps *cutil.PageState) (string, error) {
+	controller.Act(searchKey, rc, func(as *app.State, ps *cutil.PageState) (string, error) {
 		q := string(rc.URI().QueryArgs().Peek("q"))
 		params := &search.Params{Q: q, PS: ps.Params}
 		results, errs := search.Search(ps.Context, params, as, ps)
-		ps.Title = "Search Results"
+		ps.SetTitleAndData("Search Results", results)
 		if q != "" {
 			ps.Title = fmt.Sprintf("[%s] %s", q, ps.Title)
 		}
 		if len(results) == 1 && results[0].URL != "" {
 			return controller.FlashAndRedir(true, "single search result found", results[0].URL, rc, ps)
 		}
-		ps.Data = results
-		ps.DefaultNavIcon = "search"
+		ps.DefaultNavIcon = searchKey
 		bc := []string{"Search||/search"}
 		if q != "" {
 			bc = append(bc, q)

app/controller/clib/socket.go

@@ -16,20 +16,20 @@ import (
 	"github.com/kyleu/npn/views/vadmin"
 )
 
+const socketIcon = "eye"
+
 func socketRoute(rc *fasthttp.RequestCtx, as *app.State, ps *cutil.PageState, path ...string) (string, error) {
 	bc := func(extra ...string) []string {
 		return append([]string{"admin", "Sockets||/admin/sockets"}, extra...)
 	}
 	if len(path) == 0 {
-		ps.Title = "Sockets"
 		chans, conns, taps := as.Services.Socket.Status()
-		ps.Data = util.ValueMap{"channels": chans, "connections": conns}
+		ps.SetTitleAndData("Sockets", util.ValueMap{"channels": chans, "connections": conns})
 		return controller.Render(rc, as, &vadmin.Sockets{Channels: chans, Connections: conns, Taps: taps}, ps, bc()...)
 	}
 	switch path[0] {
 	case "tap":
-		ps.Title = "WebSocket Tap"
-		ps.Data = ps.Title
+		ps.SetTitleAndData("WebSocket Tap", "tap")
 		ps.DefaultNavIcon = "search"
 		return controller.Render(rc, as, &vadmin.SocketTap{}, ps, bc("Tap")...)
 	case "tap-socket":
@@ -45,7 +45,7 @@ func socketRoute(rc *fasthttp.RequestCtx, as *app.State, ps *cutil.PageState, pa
 		ch := as.Services.Socket.GetChannel(path[1])
 		members := as.Services.Socket.GetAllMembers(path[1])
 		ps.Data = ch
-		ps.DefaultNavIcon = "eye"
+		ps.DefaultNavIcon = socketIcon
 		return controller.Render(rc, as, &vadmin.Channel{Channel: ch, Members: members}, ps, bc("Channel", ch.Key)...)
 	case "conn":
 		if len(path) == 0 {
@@ -61,7 +61,7 @@ func socketRoute(rc *fasthttp.RequestCtx, as *app.State, ps *cutil.PageState, pa
 				return "", errors.Errorf("no connection with ID [%s]", id.String())
 			}
 			ps.Data = c
-			ps.DefaultNavIcon = "eye"
+			ps.DefaultNavIcon = socketIcon
 			return controller.Render(rc, as, &vadmin.Connection{Connection: c}, ps, bc("Connection", c.ID.String())...)
 		}
 		frm, _ := cutil.ParseForm(rc)

app/controller/clib/theme.go

@@ -14,12 +14,13 @@ import (
 	"github.com/kyleu/npn/views/vtheme"
 )
 
+const themeIcon = "gift"
+
 func ThemeList(rc *fasthttp.RequestCtx) {
 	controller.Act("theme.list", rc, func(as *app.State, ps *cutil.PageState) (string, error) {
-		ps.Title = "Themes"
 		th := as.Themes.All(ps.Logger)
-		ps.Data = th
-		ps.DefaultNavIcon = "gift"
+		ps.SetTitleAndData("Themes", th)
+		ps.DefaultNavIcon = themeIcon
 		return controller.Render(rc, as, &vtheme.List{Themes: th}, ps, "Themes||/theme")
 	})
 }
@@ -51,9 +52,8 @@ func ThemeEdit(rc *fasthttp.RequestCtx) {
 		if t == nil {
 			return "", errors.Errorf("invalid theme [%s]", key)
 		}
-		ps.Title = "Edit theme [" + t.Key + "]"
-		ps.Data = t
-		ps.DefaultNavIcon = "gift"
+		ps.SetTitleAndData("Edit theme ["+t.Key+"]", t)
+		ps.DefaultNavIcon = themeIcon
 		page := &vtheme.Edit{Theme: t, Icon: "app", Exists: as.Themes.FileExists(t.Key)}
 		return controller.Render(rc, as, page, ps, "Themes||/theme", t.Key)
 	})

app/controller/clib/themecatalog.go

@@ -30,10 +30,9 @@ func ThemeColor(rc *fasthttp.RequestCtx) {
 		if !strings.HasPrefix(col, "#") {
 			col = "#" + col
 		}
-		ps.Title = fmt.Sprintf("[%s] Theme", col)
 		th := theme.ColorTheme(col, gamut.Hex(col))
-		ps.Data = th
-		ps.DefaultNavIcon = "gift"
+		ps.SetTitleAndData(fmt.Sprintf("[%s] Theme", col), th)
+		ps.DefaultNavIcon = themeIcon
 		return controller.Render(rc, as, &vtheme.Edit{Theme: th, Icon: "app"}, ps, "Themes||/theme", col)
 	})
 }
@@ -48,9 +47,8 @@ func ThemeColorEdit(rc *fasthttp.RequestCtx) {
 			return "", errors.New("provided color must be a hex string")
 		}
 		t := theme.ColorTheme(strings.TrimPrefix(color, "#"), gamut.Hex(color))
-		ps.Title = "Edit theme [" + t.Key + "]"
-		ps.Data = t
-		ps.DefaultNavIcon = "gift"
+		ps.SetTitleAndData("Edit theme ["+t.Key+"]", t)
+		ps.DefaultNavIcon = themeIcon
 		page := &vtheme.Edit{Theme: t, Icon: "app", Exists: as.Themes.FileExists(t.Key)}
 		return controller.Render(rc, as, page, ps, "Themes||/theme", t.Key)
 	})
@@ -62,22 +60,20 @@ func ThemePalette(rc *fasthttp.RequestCtx) {
 		if err != nil {
 			return "", err
 		}
-		ps.Title = fmt.Sprintf("[%s] Themes", pal)
 		_, span, _ := telemetry.StartSpan(ps.Context, "theme:load", ps.Logger)
 		thms, err := theme.PaletteThemes(pal)
 		span.Complete()
 		if err != nil {
 			return "", err
 		}
+		ps.SetTitleAndData(fmt.Sprintf("[%s] Themes", pal), thms)
 		if string(rc.URI().QueryArgs().Peek("t")) == "go" {
 			ps.Data = strings.Join(lo.Map(thms, func(t *theme.Theme, _ int) string {
 				return t.ToGo()
 			}), util.StringDefaultLinebreak)
 			return controller.Render(rc, as, &views.Debug{}, ps, "Themes")
-		} else {
-			ps.Data = thms
 		}
-		ps.DefaultNavIcon = "gift"
+		ps.DefaultNavIcon = themeIcon
 		return controller.Render(rc, as, &vtheme.Add{Palette: pal, Themes: thms}, ps, "Themes||/theme", "Palette")
 	})
 }
@@ -103,9 +99,8 @@ func ThemePaletteEdit(rc *fasthttp.RequestCtx) {
 		if t == nil {
 			return "", errors.Errorf("invalid theme [%s] for palette [%s]", key, palette)
 		}
-		ps.Title = "Edit theme [" + t.Key + "]"
-		ps.Data = t
-		ps.DefaultNavIcon = "gift"
+		ps.SetTitleAndData("Edit theme ["+t.Key+"]", t)
+		ps.DefaultNavIcon = themeIcon
 		return controller.Render(rc, as, &vtheme.Edit{Theme: t, Icon: "app"}, ps, "Themes||/theme", t.Key)
 	})
 }

app/controller/cutil/args.go

@@ -4,6 +4,8 @@ package cutil
 import (
 	"github.com/samber/lo"
 	"github.com/valyala/fasthttp"
+
+	"github.com/kyleu/npn/app/util"
 )
 
 type Arg struct {
@@ -18,17 +20,17 @@ type Arg struct {
 type Args []*Arg
 
 type ArgResults struct {
-	Args    Args              `json:"args"`
-	Values  map[string]string `json:"values"`
-	Missing []string          `json:"missing,omitempty"`
+	Args    Args          `json:"args"`
+	Values  util.ValueMap `json:"values"`
+	Missing []string      `json:"missing,omitempty"`
 }
 
 func (a *ArgResults) HasMissing() bool {
 	return len(a.Missing) > 0
 }
 
 func CollectArgs(rc *fasthttp.RequestCtx, args Args) *ArgResults {
-	ret := make(map[string]string, len(args))
+	ret := make(util.ValueMap, len(args))
 	var missing []string
 	lo.ForEach(args, func(arg *Arg, _ int) {
 		qa := rc.URI().QueryArgs()

app/controller/cutil/page.go

@@ -180,3 +180,8 @@ func (p *PageState) ClassDecl() string {
 	classes := strings.Join(ret, " ")
 	return fmt.Sprintf(` class=%q`, classes)
 }
+
+func (p *PageState) SetTitleAndData(title string, data any) {
+	p.Title = title
+	p.Data = data
+}

app/lib/filesystem/fileloader.go

@@ -2,6 +2,7 @@
 package filesystem
 
 import (
+	"context"
 	"io"
 
 	"github.com/kyleu/npn/app/util"
@@ -16,7 +17,7 @@ type FileLoader interface {
 	FileReader(fn string) (io.Reader, error)
 	CreateDirectory(path string) error
 	WriteFile(path string, content []byte, mode FileMode, overwrite bool) error
-	FileWriter(fn string, createIfNeeded bool) (io.Writer, error)
+	FileWriter(fn string, createIfNeeded bool, appendMode bool) (io.Writer, error)
 	CopyFile(src string, tgt string) error
 	CopyRecursive(src string, tgt string, ignore []string, logger util.Logger) error
 	Move(src string, tgt string) error
@@ -33,5 +34,6 @@ type FileLoader interface {
 	IsDir(path string) bool
 	Remove(path string, logger util.Logger) error
 	RemoveRecursive(pt string, logger util.Logger) error
+	Download(ctx context.Context, url string, path string, overwrite bool, logger util.Logger) (int, error)
 	String() string
 }

app/lib/filesystem/fsdownload.go

@@ -0,0 +1,45 @@
+// Package filesystem - Content managed by Project Forge, see [projectforge.md] for details.
+package filesystem
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/kyleu/npn/app/util"
+)
+
+func (f *FileSystem) Download(ctx context.Context, url string, path string, overwrite bool, _ util.Logger) (int, error) {
+	if !strings.HasPrefix(url, "https://") {
+		return 0, errors.New("only [https] requests are supported")
+	}
+	if f.Exists(path) && !overwrite {
+		return 0, errors.Errorf("file [%s] exists", path)
+	}
+	w, err := f.FileWriter(path, true, false)
+	if err != nil {
+		return 0, errors.Wrapf(err, "unable to open file at path [%s]", path)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
+	if err != nil {
+		return 0, errors.Wrapf(err, "unable to make request for url [%s]", url)
+	}
+	rsp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return 0, errors.Wrapf(err, "unable to load url [%s]", url)
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode != http.StatusOK {
+		return 0, errors.Errorf("response from url [%s] has status [%d], expected [200]", url, rsp.StatusCode)
+	}
+	x, err := io.Copy(w, rsp.Body)
+	if err != nil {
+		return 0, errors.Wrapf(err, "unable to save response from url [%s] to file [%s]", url, path)
+	}
+	return int(x), nil
+}

app/lib/filesystem/fsread.go

@@ -53,6 +53,9 @@ func (f *FileSystem) ReadFile(path string) ([]byte, error) {
 		stat, _ := file.Stat()
 		return stat.Size(), nil
 	}()
+	if err != nil {
+		return nil, err
+	}
 
 	b, err := os.ReadFile(f.getPath(path))
 	if err != nil {

app/lib/filesystem/fswrite.go

@@ -43,13 +43,17 @@ func (f *FileSystem) WriteFile(path string, content []byte, mode FileMode, overw
 	return nil
 }
 
-func (f *FileSystem) FileWriter(fn string, createIfNeeded bool) (io.Writer, error) {
+func (f *FileSystem) FileWriter(fn string, createIfNeeded bool, appendMode bool) (io.Writer, error) {
 	p := f.getPath(fn)
 	if createIfNeeded && !f.Exists(p) {
 		_, err := f.f.Create(p)
 		if err != nil {
 			return nil, err
 		}
 	}
-	return f.f.OpenFile(p, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
+	mode := os.O_WRONLY
+	if appendMode {
+		mode = os.O_APPEND | os.O_WRONLY
+	}
+	return f.f.OpenFile(p, mode, os.ModeAppend)
 }