From a7b15723be0df2ab3d253eb4776cd21b01c2623c Mon Sep 17 00:00:00 2001 From: Kyle Unverferth Date: Sat, 12 Oct 2024 11:01:32 -0600 Subject: [PATCH] Utility function improvements, JSON cleanups --- app/controller/clib/sandbox.go | 5 ++++ app/controller/cutil/session.go | 3 +- app/enum/memberstatus.go | 4 +-- app/enum/modelservice.go | 4 +-- app/enum/sessionstatus.go | 4 +-- app/lib/filesystem/fsignore.go | 3 ++ app/lib/theme/service.go | 3 +- app/lib/websocket/socket.go | 6 +--- app/util/array.go | 7 +++++ app/util/async.go | 50 +++++++++++++++++++++++++-------- app/util/parsearray.go | 6 ++-- 11 files changed, 65 insertions(+), 30 deletions(-) diff --git a/app/controller/clib/sandbox.go b/app/controller/clib/sandbox.go index e2417adc..5326ca4b 100644 --- a/app/controller/clib/sandbox.go +++ b/app/controller/clib/sandbox.go @@ -8,11 +8,16 @@ import ( "github.com/kyleu/rituals/app/controller/cutil" "github.com/kyleu/rituals/app/lib/sandbox" "github.com/kyleu/rituals/app/lib/telemetry" + "github.com/kyleu/rituals/views" "github.com/kyleu/rituals/views/vsandbox" ) func SandboxList(w http.ResponseWriter, r *http.Request) { controller.Act("sandbox.list", w, r, func(as *app.State, ps *cutil.PageState) (string, error) { + if title := r.URL.Query().Get("title"); title != "" { + ps.SetTitleAndData(title, title) + return controller.Render(r, as, &views.Debug{}, ps, title) + } ps.SetTitleAndData("Sandboxes", sandbox.AllSandboxes) return controller.Render(r, as, &vsandbox.List{}, ps, "sandbox") }) diff --git a/app/controller/cutil/session.go b/app/controller/cutil/session.go index 4e293720..b52cfe53 100644 --- a/app/controller/cutil/session.go +++ b/app/controller/cutil/session.go @@ -130,8 +130,7 @@ func loadProfile(session util.ValueMap) (*user.Profile, error) { } s = util.ToJSON(m) } - p := &user.Profile{} - err := util.FromJSON([]byte(s), p) + p, err := util.FromJSONObj[*user.Profile]([]byte(s)) if err != nil { return nil, err } diff --git a/app/enum/memberstatus.go b/app/enum/memberstatus.go index 58d926c9..a166f3fa 100644 --- a/app/enum/memberstatus.go +++ b/app/enum/memberstatus.go @@ -43,8 +43,8 @@ func (m MemberStatus) MarshalJSON() ([]byte, error) { } func (m *MemberStatus) UnmarshalJSON(data []byte) error { - var key string - if err := util.FromJSON(data, &key); err != nil { + key, err := util.FromJSONString(data) + if err != nil { return err } *m = AllMemberStatuses.Get(key, nil) diff --git a/app/enum/modelservice.go b/app/enum/modelservice.go index 76443485..c7d60688 100644 --- a/app/enum/modelservice.go +++ b/app/enum/modelservice.go @@ -49,8 +49,8 @@ func (m ModelService) MarshalJSON() ([]byte, error) { } func (m *ModelService) UnmarshalJSON(data []byte) error { - var key string - if err := util.FromJSON(data, &key); err != nil { + key, err := util.FromJSONString(data) + if err != nil { return err } *m = AllModelServices.Get(key, nil) diff --git a/app/enum/sessionstatus.go b/app/enum/sessionstatus.go index 3e445616..7178c341 100644 --- a/app/enum/sessionstatus.go +++ b/app/enum/sessionstatus.go @@ -43,8 +43,8 @@ func (s SessionStatus) MarshalJSON() ([]byte, error) { } func (s *SessionStatus) UnmarshalJSON(data []byte) error { - var key string - if err := util.FromJSON(data, &key); err != nil { + key, err := util.FromJSONString(data) + if err != nil { return err } *s = AllSessionStatuses.Get(key, nil) diff --git a/app/lib/filesystem/fsignore.go b/app/lib/filesystem/fsignore.go index 9407e366..158b044e 100644 --- a/app/lib/filesystem/fsignore.go +++ b/app/lib/filesystem/fsignore.go @@ -9,6 +9,9 @@ import ( var defaultIgnore = []string{".DS_Store$", "^.git/", "^.idea/", "^build/", "^client/node_modules", ".html.go$", ".sql.go$"} func buildIgnore(ign []string) []string { + if len(ign) == 1 && ign[0] == "-" { + return nil + } ret := util.NewStringSlice(append([]string{}, defaultIgnore...)) ret.Push(ign...) return ret.Slice diff --git a/app/lib/theme/service.go b/app/lib/theme/service.go index e537d5a9..b134a845 100644 --- a/app/lib/theme/service.go +++ b/app/lib/theme/service.go @@ -64,12 +64,11 @@ func (s *Service) loadIfNeeded(logger util.Logger) { s.cache = Themes{Default} if s.files.IsDir(s.root) { lo.ForEach(s.files.ListJSON(s.root, nil, true, logger), func(key string, _ int) { - t := &Theme{} b, err := s.files.ReadFile(filepath.Join(s.root, key+util.ExtJSON)) if err != nil { logger.Warnf("can't load theme [%s]: %+v", key, err) } - err = util.FromJSON(b, t) + t, err := util.FromJSONObj[*Theme](b) if err != nil { logger.Warnf("can't load theme [%s]: %+v", key, err) } diff --git a/app/lib/websocket/socket.go b/app/lib/websocket/socket.go index 4ad1fb76..b5ec7aeb 100644 --- a/app/lib/websocket/socket.go +++ b/app/lib/websocket/socket.go @@ -64,9 +64,6 @@ func (s *Service) WriteChannel(message *Message, logger util.Logger, except ...u return nil } json := util.ToJSON(message) - if size := len(conns.ConnIDs) - len(except); size > 0 { - logger.Debugf("sending message [%v::%v] to [%v] connections", message.Channel, message.Cmd, size) - } lo.ForEach(conns.ConnIDs, func(conn uuid.UUID, _ int) { if !lo.Contains(except, conn) { connID := conn @@ -138,8 +135,7 @@ func ReadSocketLoop(connID uuid.UUID, sock *websocket.Conn, onMessage func(m *Me } return errors.Wrapf(err, "error processing socket read loop for connection [%s]", connID.String()) } - m := &Message{} - err = util.FromJSON(message, m) + m, err := util.FromJSONObj[*Message](message) if err != nil { return errors.Wrap(err, "error decoding websocket message") } diff --git a/app/util/array.go b/app/util/array.go index 351419aa..dd29b53c 100644 --- a/app/util/array.go +++ b/app/util/array.go @@ -106,6 +106,13 @@ func ArrayRemoveNil[T any](x []*T) []*T { }) } +func ArrayRemoveEmpty[T comparable](x []T) []T { + var check T + return lo.Reject(x, func(el T, _ int) bool { + return el == check + }) +} + func ArrayDereference[T any](x []*T) []T { return lo.Map(x, func(el *T, _ int) T { return lo.FromPtr(el) diff --git a/app/util/async.go b/app/util/async.go index 8d9b5c83..112044dd 100644 --- a/app/util/async.go +++ b/app/util/async.go @@ -1,6 +1,7 @@ package util import ( + "fmt" "sync" "time" @@ -8,80 +9,107 @@ import ( "github.com/samber/lo" ) -func AsyncCollect[T any, R any](items []T, f func(x T) (R, error)) ([]R, []error) { +func AsyncCollect[T any, R any](items []T, f func(x T) (R, error), loggers ...Logger) ([]R, []error) { ret := make([]R, 0, len(items)) var errs []error mu := sync.Mutex{} + size := len(items) wg := sync.WaitGroup{} - wg.Add(len(items)) + wg.Add(size) + var processed int + lo.ForEach(items, func(x T, _ int) { i := x go func() { + defer wg.Done() r, err := f(i) mu.Lock() + defer mu.Unlock() + processed++ if err == nil { ret = append(ret, r) } else { errs = append(errs, errors.Wrapf(err, "error running async function for item [%v]", i)) } - mu.Unlock() - wg.Done() + for _, logger := range loggers { + logger.Debugf("processed [%d/%d] items", processed, size) + } }() }) wg.Wait() return ret, errs } -func AsyncCollectMap[T any, K comparable, R any](items []T, k func(x T) K, f func(x T) (R, error)) (map[K]R, map[K]error) { +func AsyncCollectMap[T any, K comparable, R any](items []T, k func(x T) K, f func(x T) (R, error), loggers ...Logger) (map[K]R, map[K]error) { ret := make(map[K]R, len(items)) errs := map[K]error{} mu := sync.Mutex{} + size := len(items) wg := sync.WaitGroup{} - wg.Add(len(items)) + wg.Add(size) + var processed int + lo.ForEach(items, func(x T, _ int) { i := x go func() { + defer wg.Done() key := k(i) r, err := f(i) mu.Lock() + defer mu.Unlock() + processed++ if err == nil { ret[key] = r } else { errs[key] = errors.Wrapf(err, "error running async function for item [%v]", key) } - mu.Unlock() - wg.Done() + for _, logger := range loggers { + logger.Debugf("processed [%d/%d] items", processed, size) + } }() }) wg.Wait() return ret, errs } -func AsyncRateLimit[T any, R any](items []T, f func(x T) (R, error), maxConcurrent int, timeout time.Duration) ([]R, []error) { +func AsyncRateLimit[T any, R any](key string, items []T, f func(x T) (R, error), maxConcurrent int, timeout time.Duration, loggers ...Logger) ([]R, []error) { ret := make([]R, 0, len(items)) errs := make([]error, 0) mu := sync.Mutex{} + size := len(items) wg := sync.WaitGroup{} + wg.Add(size) + var processed int + var started int + prefix := fmt.Sprintf("[%s] ", key) + log := func(msg string, args ...any) { + for _, logger := range loggers { + logger.Debugf(prefix+msg, args...) + } + } limit := make(chan struct{}, maxConcurrent) defer close(limit) + log("starting to process [%d] items, [%d] at once)", size, maxConcurrent) for idx, item := range items { select { case limit <- struct{}{}: - wg.Add(1) go func(item T, idx int) { defer wg.Done() defer func() { <-limit }() - + started++ + log("starting to process item [%d/%d]...", started, size) r, err := f(item) mu.Lock() defer mu.Unlock() + processed++ if err == nil { ret = append(ret, r) } else { errs = append(errs, errors.Wrapf(err, "error running async function for item [%v]", item)) } + log("processed [%d/%d] items", processed, size) }(item, idx) case <-time.After(timeout): errs = append(errs, errors.Errorf("job timed out after [%v]", timeout)) diff --git a/app/util/parsearray.go b/app/util/parsearray.go index dc62e6be..e7eb87ea 100644 --- a/app/util/parsearray.go +++ b/app/util/parsearray.go @@ -14,8 +14,7 @@ func ParseArray(r any, path string, allowEmpty bool, coerce bool) ([]any, error) if strings.TrimSpace(t) == "" { return nil, nil } - var ret []any - err := FromJSON([]byte(t), &ret) + ret, err := FromJSONObj[[]any]([]byte(t)) if err != nil { if coerce { return lo.ToAnySlice(StringSplitAndTrim(t, ",")), nil @@ -27,8 +26,7 @@ func ParseArray(r any, path string, allowEmpty bool, coerce bool) ([]any, error) if len(t) == 0 { return nil, nil } - var ret []any - err := FromJSON(t, &ret) + ret, err := FromJSONObj[[]any](t) if err != nil { if coerce { return lo.ToAnySlice(StringSplitAndTrim(string(t), ",")), nil