cmd/doctree: refactor auto indexing and use directory hash

cmd/doctree/add.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	"github.com/hexops/cmder"
@@ -32,41 +33,26 @@ Examples:
 			return &cmder.UsageError{}
 		}
 		dir := flagSet.Arg(0)
-		// ctx := context.Background()
 		projectDir, err := filepath.Abs(dir)
 		if err != nil {
-			return errors.Wrap(err, "AbsoluteProjectDir")
+			return errors.Wrap(err, "AbsProjectDir")
 		}
+		autoIndexPath := filepath.Join(*dataDirFlag, "autoindex")
 
-		monitoredJsonPath := filepath.Join(*dataDirFlag, "monitored")
-
-		// Read JSON from ~/doctree/monitored
-		var monitored []schema.MonitoredDirectory
-		data, err := os.ReadFile(monitoredJsonPath)
+		// Read JSON from ~/.doctree/autoindex
+		autoIndexProjects, err := ReadAutoIndex(autoIndexPath)
 		if err != nil {
-			return errors.Wrap(err, "ReadMonitoredDirectory")
+			return err
 		}
-		json.Unmarshal(data, &monitored)
 
-		// Update the monitored list
-		monitored = append(monitored, schema.MonitoredDirectory{
-			ProjectName: *projectFlag,
-			Path:        projectDir,
+		// Update the autoIndexProjects array
+		autoIndexProjects = append(autoIndexProjects, schema.AutoIndexedProject{
+			Name: *projectFlag,
+			Path: projectDir,
+			Hash: GetDirHash(projectDir),
 		})
 
-		// Store JSON in ~/.doctree/monitored
-		// [{"projectName": "..", "path": "..."}, {...}, ...]
-		f, err := os.Create(monitoredJsonPath)
-		if err != nil {
-			return errors.Wrap(err, "Create")
-		}
-		defer f.Close()
-
-		if err := json.NewEncoder(f).Encode(monitored); err != nil {
-			return errors.Wrap(err, "Encode")
-		}
-
-		return nil
+		return WriteAutoIndex(autoIndexPath, autoIndexProjects)
 	}
 
 	// Register the command.
@@ -81,3 +67,52 @@ Examples:
 		},
 	})
 }
+
+func WriteAutoIndex(autoIndexPath string, autoindexProjects []schema.AutoIndexedProject) error {
+	// Store JSON in ~/.doctree/monitored
+	// [{"projectName": "..", "path": "..."}, {...}, ...]
+	f, err := os.Create(autoIndexPath)
+	if err != nil {
+		return errors.Wrap(err, "Create")
+	}
+	defer f.Close()
+
+	if err := json.NewEncoder(f).Encode(autoindexProjects); err != nil {
+		return errors.Wrap(err, "Encode")
+	}
+
+	return nil
+}
+
+func ReadAutoIndex(autoIndexPath string) ([]schema.AutoIndexedProject, error) {
+	var autoIndexList []schema.AutoIndexedProject
+	data, err := os.ReadFile(autoIndexPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "ReadAutoIndexFile")
+	}
+	json.Unmarshal(data, &autoIndexList)
+
+	return autoIndexList, nil
+}
+
+func GetDirHash(dir string) string {
+	// Reference: https://unix.stackexchange.com/questions/35832/how-do-i-get-the-md5-sum-of-a-directorys-contents-as-one-sum
+	tarCmd := exec.Command("tar", "-cf", "-", dir)
+	md5sumCmd := exec.Command("md5sum")
+
+	pipe, _ := tarCmd.StdoutPipe()
+	defer pipe.Close()
+
+	md5sumCmd.Stdin = pipe
+
+	// Run the tarCmd
+	err := tarCmd.Start()
+	if err != nil {
+		fmt.Printf("tar command failed with '%s'\n", err)
+		return "0"
+	}
+	// Run and get the output of md5sum
+	res, _ := md5sumCmd.Output()
+
+	return string(res)
+}

cmd/doctree/serve.go

@@ -20,7 +20,6 @@ import (
 	"github.com/hexops/cmder"
 	"github.com/pkg/errors"
 	"github.com/sourcegraph/doctree/doctree/indexer"
-	"github.com/sourcegraph/doctree/doctree/schema"
 	"github.com/sourcegraph/doctree/frontend"
 )
 
@@ -52,7 +51,7 @@ Examples:
 		signals := make(chan os.Signal, 1)
 		signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
 
-		go ListenToMonitoredProjects(dataDirFlag)
+		go ListenAutoIndexedProjects(dataDirFlag)
 		go Serve(*cloudModeFlag, *httpFlag, indexDataDir)
 		<-signals
 
@@ -214,28 +213,53 @@ func isParentDir(parent, child string) (bool, error) {
 	return !strings.Contains(relativePath, ".."), nil
 }
 
-func ListenToMonitoredProjects(dataDirFlag *string) error {
-	monitoredJsonPath := filepath.Join(*dataDirFlag, "monitored")
-	var monitored []schema.MonitoredDirectory
-	data, err := os.ReadFile(monitoredJsonPath)
+func ListenAutoIndexedProjects(dataDirFlag *string) error {
+	// Read the list of projects to monitor.
+	autoIndexPath := filepath.Join(*dataDirFlag, "autoindex")
+	autoindexProjects, err := ReadAutoIndex(autoIndexPath)
 	if err != nil {
-		return errors.Wrap(err, "ReadMonitoredDirectory")
+		log.Fatal(err)
 	}
-	json.Unmarshal(data, &monitored)
 
-	// Watch the directory for file changes
+	// Initialize the fsnotify watcher
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	// Configure watcher to watch all dirs mentioned in the 'monitored' file
-	for _, dir := range monitored {
-		err = watcher.Add(dir.Path)
+	// Configure watcher to watch all dirs mentioned in the 'autoindex' file
+	for i, project := range autoindexProjects {
+		if GetDirHash(project.Path) != project.Hash {
+			log.Printf("Project %s has been modified while server was down, reindexing", project.Name)
+			ctx := context.Background()
+			if err != nil {
+				log.Fatal(err)
+			}
+			RunIndexers(ctx, project.Path, dataDirFlag, &project.Name)
+
+			// Update the autoIndexedProjects array
+			autoindexProjectPtr := &autoindexProjects[i]
+			autoindexProjectPtr.Hash = GetDirHash(project.Path)
+			WriteAutoIndex(autoIndexPath, autoindexProjects)
+		}
+
+		// Add the project directory to the watcher
+		// TODO: Watch nested directories
+		err = watcher.Add(project.Path)
 		if err != nil {
 			log.Fatal(err)
 		}
-		log.Println("Watching", dir)
+		log.Println("Watching", project)
+	}
+
+	f, err := os.Create(autoIndexPath)
+	if err != nil {
+		return errors.Wrap(err, "Create")
+	}
+	defer f.Close()
+
+	if err := json.NewEncoder(f).Encode(autoindexProjects); err != nil {
+		return errors.Wrap(err, "Encode")
 	}
 
 	done := make(chan error)
@@ -246,7 +270,7 @@ func ListenToMonitoredProjects(dataDirFlag *string) error {
 			select {
 			case ev := <-watcher.Events:
 				log.Println("Event:", ev)
-				for _, dir := range monitored {
+				for _, dir := range autoindexProjects {
 					isParent, err := isParentDir(dir.Path, ev.Name)
 					if err != nil {
 						log.Println(err)
@@ -259,7 +283,7 @@ func ListenToMonitoredProjects(dataDirFlag *string) error {
 							log.Println(err)
 							return
 						}
-						RunIndexers(ctx, dir.Path, dataDirFlag, &dir.ProjectName)
+						RunIndexers(ctx, dir.Path, dataDirFlag, &dir.Name)
 						break // Only reindex for the first matching parent
 					}
 				}

doctree/indexer/golang/indexer.go

@@ -39,7 +39,7 @@ func (i *goIndexer) IndexDir(ctx context.Context, dir string) (*schema.Index, er
 		if !d.IsDir() {
 			ext := filepath.Ext(path)
 			if ext == ".go" {
-				sources = append(sources, path)
+				sources = append(sources, dir+"/"+path)
 			}
 		}
 		return nil
@@ -59,6 +59,10 @@ func (i *goIndexer) IndexDir(ctx context.Context, dir string) (*schema.Index, er
 		if err != nil {
 			return nil, errors.Wrap(err, "ReadFile")
 		}
+		path, err := filepath.Rel(dir, path)
+		if err != nil {
+			return nil, errors.Wrap(err, "RelativeFilePath")
+		}
 		files += 1
 		bytes += len(content)
 

doctree/indexer/markdown/indexer.go

@@ -50,6 +50,10 @@ func (i *markdownIndexer) IndexDir(ctx context.Context, dir string) (*schema.Ind
 		if err != nil {
 			return nil, errors.Wrap(err, "ReadFile")
 		}
+		path, err := filepath.Rel(dir, path)
+		if err != nil {
+			return nil, errors.Wrap(err, "RelativeFilePath")
+		}
 		files += 1
 		bytes += len(content)
 

doctree/indexer/python/indexer.go

@@ -57,6 +57,10 @@ func (i *pythonIndexer) IndexDir(ctx context.Context, dir string) (*schema.Index
 		if err != nil {
 			return nil, errors.Wrap(err, "ReadFile")
 		}
+		path, err := filepath.Rel(dir, path)
+		if err != nil {
+			return nil, errors.Wrap(err, "RelativeFilePath")
+		}
 		files += 1
 		bytes += len(content)
 

doctree/schema/schema.go

@@ -7,12 +7,15 @@ package schema
 // LatestVersion of the doctree schema (semver.)
 const LatestVersion = "0.0.1"
 
-type MonitoredDirectory struct {
+type AutoIndexedProject struct {
 	// Name of the project to be monitored (default value is the git repo url.)
-	ProjectName string `json:"projectName"`
+	Name string `json:"projectName"`
 
-	// Path of the directory that is to be monitored (absolute path.)
+	// Path of the project directory that is to be monitored (absolute path.)
 	Path string `json:"path"`
+
+	// Last modified timestamp of the directory.
+	Hash string `json:"hash"`
 }
 
 // Index is the top-most data structure in the doctree schema. It is produed by running a language