diff --git a/Dockerfile b/Dockerfile index 8077c497..674d8c82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ ENV GOPATH /go COPY . /go/src/github.com/hound-search/hound RUN apk update \ - && apk add go git subversion libc-dev mercurial bzr openssh \ + && apk add go git subversion libc-dev mercurial openssh \ + && go get github.com/fsnotify/fsnotify \ && go install github.com/hound-search/hound/cmds/houndd \ && apk del go \ && rm -f /var/cache/apk/* \ diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go index 43d4a304..fbd32544 100644 --- a/cmds/houndd/main.go +++ b/cmds/houndd/main.go @@ -1,11 +1,9 @@ package main import ( - "encoding/json" "flag" "fmt" "log" - "net/http" "os" "os/exec" "os/signal" @@ -14,11 +12,10 @@ import ( "strings" "syscall" + "github.com/fsnotify/fsnotify" "github.com/blang/semver" - "github.com/hound-search/hound/api" "github.com/hound-search/hound/config" "github.com/hound-search/hound/searcher" - "github.com/hound-search/hound/ui" "github.com/hound-search/hound/web" ) @@ -31,30 +28,30 @@ var ( basepath = filepath.Dir(b) ) -func makeSearchers(cfg *config.Config) (map[string]*searcher.Searcher, bool, error) { +func makeSearchers(cfg *config.Config, searchers map[string]*searcher.Searcher) (bool, error) { // Ensure we have a dbpath if _, err := os.Stat(cfg.DbPath); err != nil { if err := os.MkdirAll(cfg.DbPath, os.ModePerm); err != nil { - return nil, false, err + return false, err } } - searchers, errs, err := searcher.MakeAll(cfg) + errs, err := searcher.MakeAll(cfg, searchers) if err != nil { - return nil, false, err + return false, err } if len(errs) > 0 { // NOTE: This mutates the original config so the repos // are not even seen by other code paths. - for name, _ := range errs { + for name := range errs { delete(cfg.Repos, name) } - return searchers, false, nil + return false, nil } - return searchers, true, nil + return true, nil } func handleShutdown(shutdownCh <-chan os.Signal, searchers map[string]*searcher.Searcher) { @@ -79,42 +76,6 @@ func registerShutdownSignal() <-chan os.Signal { return shutdownCh } -func makeTemplateData(cfg *config.Config) (interface{}, error) { - var data struct { - ReposAsJson string - } - - res := map[string]*config.Repo{} - for name, repo := range cfg.Repos { - res[name] = repo - } - - b, err := json.Marshal(res) - if err != nil { - return nil, err - } - - data.ReposAsJson = string(b) - return &data, nil -} - -func runHttp( - addr string, - dev bool, - cfg *config.Config, - idx map[string]*searcher.Searcher) error { - m := http.DefaultServeMux - - h, err := ui.Content(dev, cfg) - if err != nil { - return err - } - - m.Handle("/", h) - api.Setup(m, idx) - return http.ListenAndServe(addr, m) -} - func getVersion() semver.Version { return semver.Version{ Major: 0, @@ -138,31 +99,41 @@ func main() { if *flagVer { fmt.Printf("houndd v%s", getVersion()) os.Exit(0) - } + } + idx := make(map[string]*searcher.Searcher) + var cfg config.Config - if err := cfg.LoadFromFile(*flagConf); err != nil { - panic(err) + + loadConfig := func() { + if err := cfg.LoadFromFile(*flagConf); err != nil { + panic(err) + } + // It's not safe to be killed during makeSearchers, so register the + // shutdown signal here and defer processing it until we are ready. + shutdownCh := registerShutdownSignal() + ok, err := makeSearchers(&cfg, idx) + if err != nil { + log.Panic(err) + } + if !ok { + info_log.Println("Some repos failed to index, see output above") + } else { + info_log.Println("All indexes built!") + } + handleShutdown(shutdownCh, idx) } + loadConfig() + + // watch for config file changes + configWatcher := config.NewWatcher(*flagConf) + configWatcher.OnChange(func(fsnotify.Event) { + loadConfig() + }) // Start the web server on a background routine. ws := web.Start(&cfg, *flagAddr, *flagDev) - // It's not safe to be killed during makeSearchers, so register the - // shutdown signal here and defer processing it until we are ready. - shutdownCh := registerShutdownSignal() - idx, ok, err := makeSearchers(&cfg) - if err != nil { - log.Panic(err) - } - if !ok { - info_log.Println("Some repos failed to index, see output above") - } else { - info_log.Println("All indexes built!") - } - - handleShutdown(shutdownCh, idx) - host := *flagAddr if strings.HasPrefix(host, ":") { host = "localhost" + host @@ -174,8 +145,7 @@ func main() { webpack.Dir = basepath + "/../../" webpack.Stdout = os.Stdout webpack.Stderr = os.Stderr - err = webpack.Start() - if err != nil { + if err := webpack.Start(); err != nil { error_log.Println(err) } } diff --git a/config/watcher.go b/config/watcher.go new file mode 100644 index 00000000..bd3a0486 --- /dev/null +++ b/config/watcher.go @@ -0,0 +1,79 @@ +package config + +import ( + "log" + "sync" + + "github.com/fsnotify/fsnotify" +) + +// WatcherListenerFunc defines the signature for listner functions +type WatcherListenerFunc func(fsnotify.Event) + +// Watcher watches for configuration updates and provides hooks for +// triggering post events +type Watcher struct { + listeners []WatcherListenerFunc +} + +// NewWatcher returns a new file watcher +func NewWatcher(cfgPath string) *Watcher { + log.Printf("setting up watcher for %s", cfgPath) + w := Watcher{} + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Panic(err) + } + defer watcher.Close() + // Event listener setup + eventWG := sync.WaitGroup{} + eventWG.Add(1) + go func() { + defer eventWG.Done() + for { + select { + case event, ok := <-watcher.Events: + if !ok { + // events channel is closed + log.Printf("error: events channel is closed\n") + return + } + // only trigger on creates and writes of the watched config file + if event.Name == cfgPath && event.Op&fsnotify.Write == fsnotify.Write { + log.Printf("change in config file (%s) detected\n", cfgPath) + for _, listener := range w.listeners { + listener(event) + } + } + case err, ok := <-watcher.Errors: + if !ok { + // errors channel is closed + log.Printf("error: errors channel is closed\n") + return + } + log.Println("error:", err) + return + } + } + }() + // add config file + if err := watcher.Add(cfgPath); err != nil { + log.Fatalf("failed to watch %s", cfgPath) + } + // setup is complete + wg.Done() + // wait for the event listener to complete before exiting + eventWG.Wait() + }() + // wait for watcher setup to complete + wg.Wait() + return &w +} + +// OnChange registers a listener function to be called if a file changes +func (w *Watcher) OnChange(listener WatcherListenerFunc) { + w.listeners = append(w.listeners, listener) +} diff --git a/searcher/searcher.go b/searcher/searcher.go index 67727b9d..a5388cbc 100644 --- a/searcher/searcher.go +++ b/searcher/searcher.go @@ -52,7 +52,7 @@ type limiter chan bool */ type foundRefs struct { refs []*index.IndexRef - claimed map[*index.IndexRef]bool + claimed map[string]bool lock sync.Mutex } @@ -89,7 +89,7 @@ func (r *foundRefs) claim(ref *index.IndexRef) { r.lock.Lock() defer r.lock.Unlock() - r.claimed[ref] = true + r.claimed[ref.Dir()] = true } /** @@ -101,7 +101,7 @@ func (r *foundRefs) removeUnclaimed() error { defer r.lock.Unlock() for _, ref := range r.refs { - if r.claimed[ref] { + if r.claimed[ref.Dir()] { continue } @@ -223,7 +223,7 @@ func findExistingRefs(dbpath string) (*foundRefs, error) { return &foundRefs{ refs: refs, - claimed: map[*index.IndexRef]bool{}, + claimed: map[string]bool{}, }, nil } @@ -282,24 +282,34 @@ func init() { // occurred and no other return values are valid. If an error occurs that is specific // to a particular searcher, that searcher will not be present in the searcher map and // will have an error entry in the error map. -func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) { +func MakeAll(cfg *config.Config, searchers map[string]*Searcher) (map[string]error, error) { errs := map[string]error{} - searchers := map[string]*Searcher{} refs, err := findExistingRefs(cfg.DbPath) if err != nil { - return nil, nil, err + return nil, err } lim := makeLimiter(cfg.MaxConcurrentIndexers) - n := len(cfg.Repos) + n := 0 + for name := range cfg.Repos { + if s, ok := searchers[name]; ok { + // claim any already running searcher refs so that they don't get removed + refs.claim(s.idx.Ref) + continue + } + n++ + } // Channel to receive the results from newSearcherConcurrent function. resultCh := make(chan searcherResult, n) // Start new searchers for all repos in different go routines while // respecting cfg.MaxConcurrentIndexers. for name, repo := range cfg.Repos { + if _, ok := searchers[name]; ok { + continue + } go newSearcherConcurrent(cfg.DbPath, name, repo, refs, lim, resultCh) } @@ -315,7 +325,7 @@ func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) } if err := refs.removeUnclaimed(); err != nil { - return nil, nil, err + return nil, err } // after all the repos are in good shape, we start their polling @@ -323,7 +333,7 @@ func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) s.begin() } - return searchers, errs, nil + return errs, nil } // Creates a new Searcher that is available for searches as soon as this returns. diff --git a/ui/ui.go b/ui/ui.go index 4e623c18..f59d5d9e 100644 --- a/ui/ui.go +++ b/ui/ui.go @@ -129,7 +129,7 @@ func (h *prdHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ct := h.content[p] if ct != nil { // if so, render it - if err := renderForPrd(w, ct, h.cfg, h.cfgJson, r); err != nil { + if err := renderForPrd(w, ct, h.cfg, r); err != nil { log.Panic(err) } return @@ -143,7 +143,7 @@ func (h *prdHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Renders a templated asset in prd-mode. This strategy will embed // the sources directly in a script tag on the templated page. -func renderForPrd(w io.Writer, c *content, cfg *config.Config, cfgJson string, r *http.Request) error { +func renderForPrd(w io.Writer, c *content, cfg *config.Config, r *http.Request) error { var buf bytes.Buffer buf.WriteString("") + json, err := cfg.ToJsonString() + if err != nil { + return err + } + return c.tpl.Execute(w, map[string]interface{}{ "ReactVersion": ReactVersion, "jQueryVersion": JQueryVersion, - "ReposAsJson": cfgJson, + "ReposAsJson": json, "Title": cfg.Title, "Source": html_template.HTML(buf.String()), "Host": r.Host,