cmd/doctree: Add automatic re-indexing

cmd/doctree/add.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"path/filepath"
+
+	"github.com/hexops/cmder"
+	"github.com/pkg/errors"
+	"github.com/sourcegraph/doctree/doctree/indexer"
+)
+
+func init() {
+	const usage = `
+Examples:
+
+  Register current directory for auto-indexing:
+
+    $ doctree add .
+`
+	// Parse flags for our subcommand.
+	flagSet := flag.NewFlagSet("add", flag.ExitOnError)
+	dataDirFlag := flagSet.String("data-dir", defaultDataDir(), "where doctree stores its data")
+	projectFlag := flagSet.String("project", defaultProjectName("."), "name of the project")
+
+	// Handles calls to our subcommand.
+	handler := func(args []string) error {
+		_ = flagSet.Parse(args)
+		if flagSet.NArg() != 1 {
+			return &cmder.UsageError{}
+		}
+		dir := flagSet.Arg(0)
+		if dir != "." {
+			*projectFlag = defaultProjectName(dir)
+		}
+
+		projectPath, err := filepath.Abs(dir)
+		if err != nil {
+			return errors.Wrap(err, "projectPath")
+		}
+		autoIndexPath := filepath.Join(*dataDirFlag, "autoindex")
+
+		// Read JSON from ~/.doctree/autoindex
+		autoIndexedProjects, err := indexer.ReadAutoIndex(autoIndexPath)
+		if err != nil {
+			return err
+		}
+
+		// Update the autoIndexProjects array
+		autoIndexedProjects[projectPath] = indexer.AutoIndexedProject{
+			Name: *projectFlag,
+		}
+
+		err = indexer.WriteAutoIndex(autoIndexPath, autoIndexedProjects)
+		if err != nil {
+			return err
+		}
+
+		// Run indexers on the newly registered dir
+		ctx := context.Background()
+		return indexer.RunIndexers(ctx, projectPath, dataDirFlag, projectFlag)
+	}
+
+	// Register the command.
+	commands = append(commands, &cmder.Command{
+		FlagSet: flagSet,
+		Aliases: []string{},
+		Handler: handler,
+		UsageFunc: func() {
+			fmt.Fprintf(flag.CommandLine.Output(), "Usage of 'doctree %s':\n", flagSet.Name())
+			flagSet.PrintDefaults()
+			fmt.Fprintf(flag.CommandLine.Output(), "%s", usage)
+		},
+	})
+}

cmd/doctree/index.go

@@ -4,14 +4,11 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"path/filepath"
-	"time"
 
-	"github.com/hashicorp/go-multierror"
 	"github.com/hexops/cmder"
-	"github.com/pkg/errors"
 
 	// Register language indexers.
+
 	"github.com/sourcegraph/doctree/doctree/indexer"
 	_ "github.com/sourcegraph/doctree/doctree/indexer/golang"
 	_ "github.com/sourcegraph/doctree/doctree/indexer/markdown"
@@ -31,7 +28,7 @@ Examples:
 	// Parse flags for our subcommand.
 	flagSet := flag.NewFlagSet("index", flag.ExitOnError)
 	dataDirFlag := flagSet.String("data-dir", defaultDataDir(), "where doctree stores its data")
-	projectFlag := flagSet.String("project", defaultProjectName(), "name of the project")
+	projectFlag := flagSet.String("project", defaultProjectName("."), "name of the project")
 
 	// Handles calls to our subcommand.
 	handler := func(args []string) error {
@@ -40,30 +37,12 @@ Examples:
 			return &cmder.UsageError{}
 		}
 		dir := flagSet.Arg(0)
-
-		ctx := context.Background()
-		indexes, indexErr := indexer.IndexDir(ctx, dir)
-		for _, index := range indexes {
-			fmt.Printf("%v: indexed %v files (%v bytes) in %v\n", index.Language.ID, index.NumFiles, index.NumBytes, time.Duration(index.DurationSeconds*float64(time.Second)).Round(time.Millisecond))
+		if dir != "." {
+			*projectFlag = defaultProjectName(dir)
 		}
 
-		indexDataDir := filepath.Join(*dataDirFlag, "index")
-		writeErr := indexer.WriteIndexes(*projectFlag, indexDataDir, indexes)
-		if indexErr != nil && writeErr != nil {
-			return multierror.Append(indexErr, writeErr)
-		}
-		if indexErr != nil {
-			return indexErr
-		}
-		if writeErr != nil {
-			return writeErr
-		}
-
-		err := indexer.IndexForSearch(*projectFlag, indexDataDir, indexes)
-		if err != nil {
-			return errors.Wrap(err, "IndexForSearch")
-		}
-		return nil
+		ctx := context.Background()
+		return indexer.RunIndexers(ctx, dir, dataDirFlag, projectFlag)
 	}
 
 	// Register the command.

cmd/doctree/main.go

@@ -18,6 +18,7 @@ Usage:
 The commands are:
 	serve    runs a doctree server
 	index    index a directory
+	add      (EXPERIMENTAL) register a directory for auto-indexing
 
 Use "doctree <command> -h" for more information about a command.
 `

cmd/doctree/serve.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -9,9 +10,12 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"os/signal"
 	"path/filepath"
+	"syscall"
 
 	"github.com/NYTimes/gziphandler"
+	"github.com/fsnotify/fsnotify"
 	"github.com/hexops/cmder"
 	"github.com/pkg/errors"
 	"github.com/sourcegraph/doctree/doctree/indexer"
@@ -42,7 +46,20 @@ Examples:
 	handler := func(args []string) error {
 		_ = flagSet.Parse(args)
 		indexDataDir := filepath.Join(*dataDirFlag, "index")
-		return Serve(*cloudModeFlag, *httpFlag, indexDataDir)
+
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
+
+		go Serve(*cloudModeFlag, *httpFlag, indexDataDir)
+		go func() {
+			err := ListenAutoIndexedProjects(dataDirFlag)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}()
+		<-signals
+
+		return nil
 	}
 
 	// Register the command.
@@ -59,7 +76,7 @@ Examples:
 }
 
 // Serve an HTTP server on the given addr.
-func Serve(cloudMode bool, addr, indexDataDir string) error {
+func Serve(cloudMode bool, addr, indexDataDir string) {
 	log.Printf("Listening on %s", addr)
 	mux := http.NewServeMux()
 	mux.Handle("/", frontendHandler())
@@ -144,9 +161,8 @@ func Serve(cloudMode bool, addr, indexDataDir string) error {
 	}))
 	muxWithGzip := gziphandler.GzipHandler(mux)
 	if err := http.ListenAndServe(addr, muxWithGzip); err != nil {
-		return errors.Wrap(err, "ListenAndServe")
+		log.Fatal(errors.Wrap(err, "ListenAndServe"))
 	}
-	return nil
 }
 
 func frontendHandler() http.Handler {
@@ -191,3 +207,74 @@ func frontendHandler() http.Handler {
 		fileServer.ServeHTTP(w, req)
 	})
 }
+
+func ListenAutoIndexedProjects(dataDirFlag *string) error {
+	// Read the list of projects to monitor.
+	autoIndexPath := filepath.Join(*dataDirFlag, "autoindex")
+	autoindexedProjects, err := indexer.ReadAutoIndex(autoIndexPath)
+	if err != nil {
+		return err
+	}
+
+	// Initialize the fsnotify watcher
+	// TODO: Watch ~/.doctree/autoindex
+	// to re-index newly added projects on the fly?
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return err
+	}
+	defer watcher.Close()
+
+	// Configure watcher to watch all dirs mentioned in the 'autoindex' file
+	for projectPath := range autoindexedProjects {
+		// Add the project directory to the watcher
+		// TODO: Check if the project changed
+		// while the server wasn't running.
+		err = recursiveWatch(watcher, projectPath)
+		if err != nil {
+			return err
+		}
+		log.Println("Watching", projectPath)
+	}
+
+	err = indexer.WriteAutoIndex(autoIndexPath, autoindexedProjects)
+	if err != nil {
+		return err
+	}
+	done := make(chan error)
+
+	// Process events
+	go func() {
+		for {
+			select {
+			case ev := <-watcher.Events:
+				log.Println("Event:", ev)
+				for projectPath, project := range autoindexedProjects {
+					isParent, err := isParentDir(projectPath, ev.Name)
+					if err != nil {
+						log.Println(err)
+						return
+					}
+					if isParent {
+						log.Println("Reindexing", projectPath)
+						ctx := context.Background()
+						if err != nil {
+							log.Println(err)
+							return
+						}
+						err := indexer.RunIndexers(ctx, projectPath, dataDirFlag, &project.Name)
+						if err != nil {
+							log.Fatal(err)
+						}
+						break // Only reindex for the first matching parent
+					}
+				}
+			case err := <-watcher.Errors:
+				log.Println("Error:", err)
+			}
+		}
+	}()
+	<-done
+
+	return nil
+}

cmd/doctree/util.go

@@ -8,12 +8,17 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/fsnotify/fsnotify"
 	"github.com/pkg/errors"
 )
 
 func getGitURIForFile(dir string) (string, error) {
 	cmd := exec.Command("git", "config", "--get", "remote.origin.url")
-	cmd.Dir = filepath.Dir(dir)
+	absDir, err := filepath.Abs(dir)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to get absolute path for %s", dir)
+	}
+	cmd.Dir = absDir
 	out, err := cmd.Output()
 	if err != nil {
 		return "", errors.Wrapf(err, "git config --get remote.origin.url (pwd=%s)", cmd.Dir)
@@ -51,14 +56,38 @@ func defaultDataDir() string {
 	return filepath.Join(home, ".doctree")
 }
 
-func defaultProjectName() string {
-	uri, err := getGitURIForFile(".")
+func defaultProjectName(defaultDir string) string {
+	uri, err := getGitURIForFile(defaultDir)
 	if err != nil {
-		absDir, err := filepath.Abs(".")
+		absDir, err := filepath.Abs(defaultDir)
 		if err != nil {
 			return ""
 		}
 		return absDir
 	}
 	return uri
 }
+
+func isParentDir(parent, child string) (bool, error) {
+	relativePath, err := filepath.Rel(parent, child)
+	if err != nil {
+		return false, err
+	}
+	return !strings.Contains(relativePath, ".."), nil
+}
+
+// Recursively watch a directory
+func recursiveWatch(watcher *fsnotify.Watcher, dir string) error {
+	err := filepath.Walk(dir, func(walkPath string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if fi.IsDir() && !strings.HasPrefix(fi.Name(), ".") { // file is directory and isn't hidden
+			if err = watcher.Add(walkPath); err != nil {
+				return errors.Wrap(err, "watcher.Add")
+			}
+		}
+		return nil
+	})
+	return err
+}

doctree/indexer/cli.go

@@ -0,0 +1,45 @@
+package indexer
+
+import (
+	"encoding/json"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+// Write autoindexedProjects as JSON in the provided filepath.
+func WriteAutoIndex(path string, autoindexedProjects map[string]AutoIndexedProject) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return errors.Wrap(err, "Create")
+	}
+	defer f.Close()
+
+	if err := json.NewEncoder(f).Encode(autoindexedProjects); err != nil {
+		return errors.Wrap(err, "Encode")
+	}
+
+	return nil
+}
+
+// Read autoindexedProjects array from the provided filepath.
+func ReadAutoIndex(path string) (map[string]AutoIndexedProject, error) {
+	autoIndexedProjects := make(map[string]AutoIndexedProject)
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			_, err := os.Create(path)
+			if err != nil {
+				return nil, errors.Wrap(err, "CreateAutoIndexFile")
+			}
+			return autoIndexedProjects, nil
+		}
+		return nil, errors.Wrap(err, "ReadAutoIndexFile")
+	}
+	err = json.Unmarshal(data, &autoIndexedProjects)
+	if err != nil {
+		return nil, errors.Wrap(err, "ParseAutoIndexFile")
+	}
+
+	return autoIndexedProjects, nil
+}