Merge branch 'main' into refinements

JackalLabs · Sep 20, 2023 · e772fba · e772fba
2 parents 2cdb810 + 6dc1b21
commit e772fba
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 27 deletions.
diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/JackalLabs/jackalgo v0.0.3
-	github.com/rs/cors v1.9.0
+	github.com/rs/cors v1.10.0
 	github.com/uptrace/bunrouter v1.0.20
 )
 

diff --git a/go.sum b/go.sum
@@ -892,6 +892,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=
 github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/rs/cors v1.10.0 h1:62NOS1h+r8p1mW6FM0FSB0exioXLhd/sh15KpjWBZ+8=
+github.com/rs/cors v1.10.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

diff --git a/japicore/routeHandling.go b/japicore/routeHandling.go
@@ -48,6 +48,8 @@ func VersionHandler() bunrouter.HandlerFunc {
 
 func ImportHandler(fileIo *file_io_handler.FileIoHandler, queue *ScrapeQueue) bunrouter.HandlerFunc {
 	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_BULK_ROOT", "s/JAPI/Bulk")
+
 		var data fileScrape
 		source := req.Header.Get("J-Source-Path")
 
@@ -61,7 +63,7 @@ func ImportHandler(fileIo *file_io_handler.FileIoHandler, queue *ScrapeQueue) bu
 
 		for _, target := range data.Targets {
 			wg.Add(1)
-			queue.Push(fileIo, w, &wg, "bulk", target, source)
+			queue.Push(fileIo, w, &wg, operatingRoot, target, source)
 		}
 
 		wg.Wait()
@@ -163,6 +165,7 @@ func UploadHandler(fileIo *file_io_handler.FileIoHandler, queue *FileIoQueue) bu
 		wg.Add(1)
 		WorkingFileSize := 32 << 30
 
+		operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_OP_ROOT", "s/JAPI")
 		envSize := jutils.LoadEnvVarOrFallback("JAPI_MAX_FILE", "")
 		if len(envSize) > 0 {
 			envParse, err := strconv.Atoi(envSize)
@@ -173,8 +176,6 @@ func UploadHandler(fileIo *file_io_handler.FileIoHandler, queue *FileIoQueue) bu
 		}
 		MaxFileSize := int64(WorkingFileSize)
 
-		operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_OP_ROOT", "s/JAPI")
-
 		// ParseMultipartForm parses a request body as multipart/form-data
 		err := req.ParseMultipartForm(MaxFileSize) // MAX file size lives here
 		if err != nil {

diff --git a/japicore/scrapeQueue.go b/japicore/scrapeQueue.go
@@ -25,13 +25,13 @@ func NewScrapeQueue(fIQueue *FileIoQueue) *ScrapeQueue {
 }
 
 type scrapee struct {
-	fileIo     *file_io_handler.FileIoHandler
-	w          http.ResponseWriter
-	wg         *sync.WaitGroup
-	err        error
-	pathSelect string
-	filename   string
-	host       string
+	fileIo   *file_io_handler.FileIoHandler
+	w        http.ResponseWriter
+	wg       *sync.WaitGroup
+	err      error
+	destPath string
+	filename string
+	host     string
 }
 
 func (m *scrapee) Error() error {
@@ -62,14 +62,14 @@ func (q *ScrapeQueue) isEmpty() bool {
 	return len(q.scrapees) == 0
 }
 
-func (q *ScrapeQueue) Push(fileIo *file_io_handler.FileIoHandler, w http.ResponseWriter, wg *sync.WaitGroup, pathSelect string, filename string, host string) *scrapee {
+func (q *ScrapeQueue) Push(fileIo *file_io_handler.FileIoHandler, w http.ResponseWriter, wg *sync.WaitGroup, destPath string, filename string, host string) *scrapee {
 	m := scrapee{
-		fileIo:     fileIo,
-		w:          w,
-		wg:         wg,
-		pathSelect: pathSelect,
-		filename:   filename,
-		host:       host,
+		fileIo:   fileIo,
+		w:        w,
+		wg:       wg,
+		destPath: destPath,
+		filename: filename,
+		host:     host,
 	}
 
 	q.scrapees = append(q.scrapees, &m)
@@ -96,7 +96,7 @@ func (q *ScrapeQueue) listenOnce() {
 		return
 	}
 
-	fid := processUpload(w, scrapee.fileIo, byteBuffer.Bytes(), filename, scrapee.pathSelect, q.loadFIQueue())
+	fid := processUpload(w, scrapee.fileIo, byteBuffer.Bytes(), filename, scrapee.destPath, q.loadFIQueue())
 	if len(fid) == 0 {
 		warning := fmt.Sprintf("Failed to get FID for %s", filename)
 		scrapee.err = jutils.ProcessCustomHttpError("processUpload", warning, 500, w)

diff --git a/jutils/errors.go b/jutils/errors.go
@@ -9,19 +9,14 @@ import (
 
 func ProcessError(block string, caughtError error) {
 	fmt.Printf("***** Error in block: %s *****\n", block)
+	fmt.Printf("***** Stamp: %s *****\n", FriendlyTimestamp())
 	fmt.Println(caughtError)
 	fmt.Println("***** End Error Report *****")
 }
 
 func ProcessHttpError(block string, caughtError error, eCode int, w http.ResponseWriter) {
-	fmt.Printf("***** Error in block: %s *****\n", block)
-	fmt.Println(caughtError)
-	fmt.Println("***** End Error Report *****")
-	w.WriteHeader(eCode)
-	_, err := w.Write([]byte(caughtError.Error()))
-	if err != nil {
-		ProcessError(fmt.Sprintf("processHttpPostError for %s", block), err)
-	}
+	ProcessError(block, caughtError)
+	http.Error(w, caughtError.Error(), eCode)
 }
 
 func ProcessCustomHttpError(block string, customError string, eCode int, w http.ResponseWriter) error {

diff --git a/jutils/time.go b/jutils/time.go
@@ -0,0 +1,17 @@
+package jutils
+
+import (
+	"fmt"
+	"time"
+)
+
+func FriendlyTimestamp() string {
+	currentTime := time.Now()
+	return fmt.Sprintf("%d-%d-%d %d:%d:%d\n",
+		currentTime.Year(),
+		currentTime.Month(),
+		currentTime.Day(),
+		currentTime.Hour(),
+		currentTime.Minute(),
+		currentTime.Second())
+}