From 1a1233a8c723c84249538372479e368eca7b8bf8 Mon Sep 17 00:00:00 2001 From: Paolo Fabio Zaino Date: Fri, 16 Feb 2024 13:10:02 +0000 Subject: [PATCH 1/3] Code refactoring to reduce code repetition in the API --- services/api/helpers.go | 51 ++++++++++ services/api/main.go | 207 +++++----------------------------------- 2 files changed, 74 insertions(+), 184 deletions(-) create mode 100644 services/api/helpers.go diff --git a/services/api/helpers.go b/services/api/helpers.go new file mode 100644 index 00000000..97fd0c5b --- /dev/null +++ b/services/api/helpers.go @@ -0,0 +1,51 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + + cmn "github.com/pzaino/thecrowler/pkg/common" +) + +// handleErrorAndRespond encapsulates common error handling and JSON response logic. +func handleErrorAndRespond(w http.ResponseWriter, err error, results interface{}, errMsg string, errCode int) { + fmt.Printf("Results: %v\n", results) + if err != nil { + cmn.DebugMsg(cmn.DbgLvlDebug3, errMsg, err) + http.Error(w, err.Error(), errCode) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(results); err != nil { + // Log the error and send a generic error message to the client + cmn.DebugMsg(cmn.DbgLvlDebug3, "Error encoding JSON response: %v", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + } +} + +// extractQueryOrBody extracts the query parameter for GET requests or the body for POST requests. +func extractQueryOrBody(r *http.Request) (string, error) { + if r.Method == "POST" { + body, err := io.ReadAll(r.Body) + defer r.Body.Close() + if err != nil { + return "", err + } + return string(body), nil + } else { + query := r.URL.Query().Get("q") + if query == "" { + return "", fmt.Errorf("query parameter 'q' is required") + } + return query, nil + } +} + +func getQType(expr bool) int { + if expr { + return 1 + } + return 0 +} diff --git a/services/api/main.go b/services/api/main.go index 82f4b938..9fb77d51 100644 --- a/services/api/main.go +++ b/services/api/main.go @@ -16,10 +16,8 @@ package main import ( - "encoding/json" "flag" "fmt" - "io" "log" "net/http" "runtime" @@ -32,12 +30,6 @@ import ( "golang.org/x/time/rate" ) -const ( - jsonErrorPrefix = "Error encoding JSON: %v" - contentTypeLabel = "Content-Type" - contentTypeJSON = "application/json" -) - // Create a rate limiter for your application. Adjust the parameters as needed. var limiter *rate.Limiter @@ -128,213 +120,60 @@ func SecurityHeadersMiddleware(next http.Handler) http.Handler { // searchHandler handles the traditional search requests func searchHandler(w http.ResponseWriter, r *http.Request) { - // Extract query parameter - query := r.URL.Query().Get("q") - if query == "" { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in search get request") - http.Error(w, "Query parameter 'q' is required for search get request", http.StatusBadRequest) - return - } - - // Perform the search - results, err := performSearch(query) + query, err := extractQueryOrBody(r) if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing search: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) + handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in search request", http.StatusBadRequest) return } - // Respond with JSON - w.Header().Set(contentTypeLabel, contentTypeJSON) - err = json.NewEncoder(w).Encode(results) - if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + results, err := performSearch(query) + handleErrorAndRespond(w, err, results, "Error performing search: %v", http.StatusInternalServerError) } // scrImgSrchHandler handles the search requests for screenshot images func scrImgSrchHandler(w http.ResponseWriter, r *http.Request) { - var results ScreenshotResponse - var query string - var err error - if r.Method == "POST" { - // Read the JSON document from the request body - body, err := io.ReadAll(r.Body) - defer r.Body.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - query = string(body) - results, err = performScreenshotSearch(query, 0) - if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing screenshot search: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } else { - // Extract query parameter - query := r.URL.Query().Get("q") - if query == "" { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in screenshot search get request") - http.Error(w, "Query parameter 'q' is required for screenshot get request", http.StatusBadRequest) - return - } - results, err = performScreenshotSearch(query, 1) - if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Error on screenshot search: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - - // Respond with JSON - w.Header().Set(contentTypeLabel, contentTypeJSON) - err = json.NewEncoder(w).Encode(results) + query, err := extractQueryOrBody(r) if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in screenshot search request", http.StatusBadRequest) return } + results, err := performScreenshotSearch(query, getQType(r.Method != "POST")) + handleErrorAndRespond(w, err, results, "Error performing screenshot search: %v", http.StatusInternalServerError) } // netInfoHandler handles the network information requests func netInfoHandler(w http.ResponseWriter, r *http.Request) { - var results NetInfoResponse - var query string - var err error - if r.Method == "POST" { - // Read the JSON document from the request body - body, err := io.ReadAll(r.Body) - defer r.Body.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - query = string(body) - results, err = performNetInfoSearch(query, 0) - if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing netinfo search: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } else { - // Extract query parameter - query := r.URL.Query().Get("q") - if query == "" { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in netinfo search get request") - http.Error(w, "Query parameter 'q' is required for netinfo get request", http.StatusBadRequest) - return - } - results, err = performNetInfoSearch(query, 1) - if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Error on netinfo search: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - - // Respond with JSON - w.Header().Set(contentTypeLabel, contentTypeJSON) - err = json.NewEncoder(w).Encode(results) + query, err := extractQueryOrBody(r) if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in netinfo search request", http.StatusBadRequest) return } + + results, err := performNetInfoSearch(query, getQType(r.Method != "POST")) + handleErrorAndRespond(w, err, results, "Error performing netinfo search: %v", http.StatusInternalServerError) } // addSourceHandler handles the addition of new sources func addSourceHandler(w http.ResponseWriter, r *http.Request) { - var results ConsoleResponse - var query string - var err error - if r.Method == "POST" { - // Read the JSON document from the request body - body, err := io.ReadAll(r.Body) - defer r.Body.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - query = string(body) - results, err = performAddSource(query, 0) - if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing addSource: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } else { - // Extract query parameter - query := r.URL.Query().Get("q") - if query == "" { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in addSource get request") - http.Error(w, "Query parameter 'q' is required for addSource get request", http.StatusBadRequest) - return - } - results, err = performAddSource(query, 1) - if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Error on addSource request: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - - // Respond with JSON - w.Header().Set(contentTypeLabel, contentTypeJSON) - err = json.NewEncoder(w).Encode(results) + query, err := extractQueryOrBody(r) if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in addSource request", http.StatusBadRequest) return } + + results, err := performAddSource(query, getQType(r.Method != "POST")) + handleErrorAndRespond(w, err, results, "Error performing addSource: %v", http.StatusInternalServerError) } // removeSourceHandler handles the removal of sources func removeSourceHandler(w http.ResponseWriter, r *http.Request) { - var results ConsoleResponse - var query string - var err error - if r.Method == "POST" { - // Read the JSON document from the request body - body, err := io.ReadAll(r.Body) - defer r.Body.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - query = string(body) - results, err = performRemoveSource(query, 0) - if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing removeSource: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } else { - // Extract query parameter - query := r.URL.Query().Get("q") - if query == "" { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in removeSource get request") - http.Error(w, "Query parameter 'q' is required for removeSource get request", http.StatusBadRequest) - return - } - results, err = performRemoveSource(query, 1) - if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, "Error on removeSource search: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - - // Respond with JSON - w.Header().Set(contentTypeLabel, contentTypeJSON) - err = json.NewEncoder(w).Encode(results) + query, err := extractQueryOrBody(r) if err != nil { - cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in removeSource request", http.StatusBadRequest) return } + + results, err := performRemoveSource(query, getQType(r.Method != "POST")) + handleErrorAndRespond(w, err, results, "Error performing removeSource: %v", http.StatusInternalServerError) } From 2128bc0684ab96ceb29e7d212a5595ac78f12b6e Mon Sep 17 00:00:00 2001 From: Paolo Fabio Zaino Date: Fri, 16 Feb 2024 13:59:29 +0000 Subject: [PATCH 2/3] Minor code refactoring to clean up console.go a bit --- pkg/database/types.go | 4 ++++ services/api/console.go | 35 +++++++++++++++++++++++------------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/pkg/database/types.go b/pkg/database/types.go index 75ce6b7b..ff690b16 100644 --- a/pkg/database/types.go +++ b/pkg/database/types.go @@ -51,6 +51,10 @@ type SQLiteHandler struct { dbms string } +type TxHandler struct { + sql.Tx +} + // Source represents the structure of the Sources table // for a record we have decided we need to crawl. // Id represents the unique identifier of the source. diff --git a/services/api/console.go b/services/api/console.go index c54ffedc..1afc0534 100644 --- a/services/api/console.go +++ b/services/api/console.go @@ -16,6 +16,7 @@ package main import ( + "database/sql" "encoding/json" cmn "github.com/pzaino/thecrowler/pkg/common" @@ -126,15 +127,31 @@ func performRemoveSource(query string, qType int) (ConsoleResponse, error) { return ConsoleResponse{Message: "Failed to start transaction"}, err } + // Proceed with deleting the source using the obtained source_id + results, err = removeSource(tx, sourceURL) + if err != nil { + return ConsoleResponse{Message: "Failed to remove source and related data"}, err + } + + // If everything went well, commit the transaction + err = tx.Commit() + if err != nil { + return ConsoleResponse{Message: "Failed to commit transaction"}, err + } + + results.Message = "Source and related data removed successfully" + return results, nil +} + +func removeSource(tx *sql.Tx, sourceURL string) (ConsoleResponse, error) { + var results ConsoleResponse + results.Message = "Failed to remove the source" + // First, get the source_id for the given URL to ensure it exists and to use in cascading deletes if necessary var sourceID int64 - err = tx.QueryRow("SELECT source_id FROM Sources WHERE url = $1", sourceURL).Scan(&sourceID) + err := tx.QueryRow("SELECT source_id FROM Sources WHERE url = $1", sourceURL).Scan(&sourceID) if err != nil { - err2 := tx.Rollback() // Rollback in case of error - if err2 != nil { - return ConsoleResponse{Message: "Failed to find source with provided URL"}, err2 - } - return ConsoleResponse{Message: "Failed to find source with provided URL"}, err + return results, err } // Proceed with deleting the source using the obtained source_id @@ -163,12 +180,6 @@ func performRemoveSource(query string, qType int) (ConsoleResponse, error) { return ConsoleResponse{Message: "Failed to cleanup orphaned netinfo"}, err } - // If everything went well, commit the transaction - err = tx.Commit() - if err != nil { - return ConsoleResponse{Message: "Failed to commit transaction"}, err - } - results.Message = "Source and related data removed successfully" return results, nil } From 5a67feaf043becbe131b58be5c410e972fea1907 Mon Sep 17 00:00:00 2001 From: Paolo Fabio Zaino Date: Fri, 16 Feb 2024 14:51:19 +0000 Subject: [PATCH 3/3] removed a debug line from code --- services/api/helpers.go | 1 - 1 file changed, 1 deletion(-) diff --git a/services/api/helpers.go b/services/api/helpers.go index 97fd0c5b..fa33c2ad 100644 --- a/services/api/helpers.go +++ b/services/api/helpers.go @@ -11,7 +11,6 @@ import ( // handleErrorAndRespond encapsulates common error handling and JSON response logic. func handleErrorAndRespond(w http.ResponseWriter, err error, results interface{}, errMsg string, errCode int) { - fmt.Printf("Results: %v\n", results) if err != nil { cmn.DebugMsg(cmn.DbgLvlDebug3, errMsg, err) http.Error(w, err.Error(), errCode)