Skip to content

Commit

Permalink
Merge pull request #104 from pzaino/develop
Browse files Browse the repository at this point in the history 
Cumulative merge of improvements from Develop branch to Main
  • Loading branch information
@pzaino
pzaino authored Feb 16, 2024
2 parents fdf454d + 5a67fea commit bd9a133
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 196 deletions.
4 changes: 4 additions & 0 deletions pkg/database/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ type SQLiteHandler struct {
dbms string
}

type TxHandler struct {
sql.Tx
}

// Source represents the structure of the Sources table
// for a record we have decided we need to crawl.
// Id represents the unique identifier of the source.
Expand Down
35 changes: 23 additions & 12 deletions services/api/console.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package main

import (
"database/sql"
"encoding/json"

cmn "github.com/pzaino/thecrowler/pkg/common"
Expand Down Expand Up @@ -126,15 +127,31 @@ func performRemoveSource(query string, qType int) (ConsoleResponse, error) {
return ConsoleResponse{Message: "Failed to start transaction"}, err
}

// Proceed with deleting the source using the obtained source_id
results, err = removeSource(tx, sourceURL)
if err != nil {
return ConsoleResponse{Message: "Failed to remove source and related data"}, err
}

// If everything went well, commit the transaction
err = tx.Commit()
if err != nil {
return ConsoleResponse{Message: "Failed to commit transaction"}, err
}

results.Message = "Source and related data removed successfully"
return results, nil
}

func removeSource(tx *sql.Tx, sourceURL string) (ConsoleResponse, error) {
var results ConsoleResponse
results.Message = "Failed to remove the source"

// First, get the source_id for the given URL to ensure it exists and to use in cascading deletes if necessary
var sourceID int64
err = tx.QueryRow("SELECT source_id FROM Sources WHERE url = $1", sourceURL).Scan(&sourceID)
err := tx.QueryRow("SELECT source_id FROM Sources WHERE url = $1", sourceURL).Scan(&sourceID)
if err != nil {
err2 := tx.Rollback() // Rollback in case of error
if err2 != nil {
return ConsoleResponse{Message: "Failed to find source with provided URL"}, err2
}
return ConsoleResponse{Message: "Failed to find source with provided URL"}, err
return results, err
}

// Proceed with deleting the source using the obtained source_id
Expand Down Expand Up @@ -163,12 +180,6 @@ func performRemoveSource(query string, qType int) (ConsoleResponse, error) {
return ConsoleResponse{Message: "Failed to cleanup orphaned netinfo"}, err
}

// If everything went well, commit the transaction
err = tx.Commit()
if err != nil {
return ConsoleResponse{Message: "Failed to commit transaction"}, err
}

results.Message = "Source and related data removed successfully"
return results, nil
}
50 changes: 50 additions & 0 deletions services/api/helpers.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package main

import (
"encoding/json"
"fmt"
"io"
"net/http"

cmn "github.com/pzaino/thecrowler/pkg/common"
)

// handleErrorAndRespond encapsulates common error handling and JSON response logic.
func handleErrorAndRespond(w http.ResponseWriter, err error, results interface{}, errMsg string, errCode int) {
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, errMsg, err)
http.Error(w, err.Error(), errCode)
return
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(results); err != nil {
// Log the error and send a generic error message to the client
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error encoding JSON response: %v", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
}

// extractQueryOrBody extracts the query parameter for GET requests or the body for POST requests.
func extractQueryOrBody(r *http.Request) (string, error) {
if r.Method == "POST" {
body, err := io.ReadAll(r.Body)
defer r.Body.Close()
if err != nil {
return "", err
}
return string(body), nil
} else {
query := r.URL.Query().Get("q")
if query == "" {
return "", fmt.Errorf("query parameter 'q' is required")
}
return query, nil
}
}

func getQType(expr bool) int {
if expr {
return 1
}
return 0
}
207 changes: 23 additions & 184 deletions services/api/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@
package main

import (
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"runtime"
Expand All @@ -32,12 +30,6 @@ import (
"golang.org/x/time/rate"
)

const (
jsonErrorPrefix = "Error encoding JSON: %v"
contentTypeLabel = "Content-Type"
contentTypeJSON = "application/json"
)

// Create a rate limiter for your application. Adjust the parameters as needed.
var limiter *rate.Limiter

Expand Down Expand Up @@ -128,213 +120,60 @@ func SecurityHeadersMiddleware(next http.Handler) http.Handler {

// searchHandler handles the traditional search requests
func searchHandler(w http.ResponseWriter, r *http.Request) {
// Extract query parameter
query := r.URL.Query().Get("q")
if query == "" {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in search get request")
http.Error(w, "Query parameter 'q' is required for search get request", http.StatusBadRequest)
return
}

// Perform the search
results, err := performSearch(query)
query, err := extractQueryOrBody(r)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing search: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in search request", http.StatusBadRequest)
return
}

// Respond with JSON
w.Header().Set(contentTypeLabel, contentTypeJSON)
err = json.NewEncoder(w).Encode(results)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
results, err := performSearch(query)
handleErrorAndRespond(w, err, results, "Error performing search: %v", http.StatusInternalServerError)
}

// scrImgSrchHandler handles the search requests for screenshot images
func scrImgSrchHandler(w http.ResponseWriter, r *http.Request) {
var results ScreenshotResponse
var query string
var err error
if r.Method == "POST" {
// Read the JSON document from the request body
body, err := io.ReadAll(r.Body)
defer r.Body.Close()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
query = string(body)
results, err = performScreenshotSearch(query, 0)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing screenshot search: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
} else {
// Extract query parameter
query := r.URL.Query().Get("q")
if query == "" {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in screenshot search get request")
http.Error(w, "Query parameter 'q' is required for screenshot get request", http.StatusBadRequest)
return
}
results, err = performScreenshotSearch(query, 1)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error on screenshot search: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}

// Respond with JSON
w.Header().Set(contentTypeLabel, contentTypeJSON)
err = json.NewEncoder(w).Encode(results)
query, err := extractQueryOrBody(r)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in screenshot search request", http.StatusBadRequest)
return
}

results, err := performScreenshotSearch(query, getQType(r.Method != "POST"))
handleErrorAndRespond(w, err, results, "Error performing screenshot search: %v", http.StatusInternalServerError)
}

// netInfoHandler handles the network information requests
func netInfoHandler(w http.ResponseWriter, r *http.Request) {
var results NetInfoResponse
var query string
var err error
if r.Method == "POST" {
// Read the JSON document from the request body
body, err := io.ReadAll(r.Body)
defer r.Body.Close()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
query = string(body)
results, err = performNetInfoSearch(query, 0)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing netinfo search: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
} else {
// Extract query parameter
query := r.URL.Query().Get("q")
if query == "" {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in netinfo search get request")
http.Error(w, "Query parameter 'q' is required for netinfo get request", http.StatusBadRequest)
return
}
results, err = performNetInfoSearch(query, 1)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error on netinfo search: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}

// Respond with JSON
w.Header().Set(contentTypeLabel, contentTypeJSON)
err = json.NewEncoder(w).Encode(results)
query, err := extractQueryOrBody(r)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in netinfo search request", http.StatusBadRequest)
return
}

results, err := performNetInfoSearch(query, getQType(r.Method != "POST"))
handleErrorAndRespond(w, err, results, "Error performing netinfo search: %v", http.StatusInternalServerError)
}

// addSourceHandler handles the addition of new sources
func addSourceHandler(w http.ResponseWriter, r *http.Request) {
var results ConsoleResponse
var query string
var err error
if r.Method == "POST" {
// Read the JSON document from the request body
body, err := io.ReadAll(r.Body)
defer r.Body.Close()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
query = string(body)
results, err = performAddSource(query, 0)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing addSource: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
} else {
// Extract query parameter
query := r.URL.Query().Get("q")
if query == "" {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in addSource get request")
http.Error(w, "Query parameter 'q' is required for addSource get request", http.StatusBadRequest)
return
}
results, err = performAddSource(query, 1)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error on addSource request: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}

// Respond with JSON
w.Header().Set(contentTypeLabel, contentTypeJSON)
err = json.NewEncoder(w).Encode(results)
query, err := extractQueryOrBody(r)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in addSource request", http.StatusBadRequest)
return
}

results, err := performAddSource(query, getQType(r.Method != "POST"))
handleErrorAndRespond(w, err, results, "Error performing addSource: %v", http.StatusInternalServerError)
}

// removeSourceHandler handles the removal of sources
func removeSourceHandler(w http.ResponseWriter, r *http.Request) {
var results ConsoleResponse
var query string
var err error
if r.Method == "POST" {
// Read the JSON document from the request body
body, err := io.ReadAll(r.Body)
defer r.Body.Close()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
query = string(body)
results, err = performRemoveSource(query, 0)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error performing removeSource: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
} else {
// Extract query parameter
query := r.URL.Query().Get("q")
if query == "" {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Missing parameter 'q' in removeSource get request")
http.Error(w, "Query parameter 'q' is required for removeSource get request", http.StatusBadRequest)
return
}
results, err = performRemoveSource(query, 1)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, "Error on removeSource search: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}

// Respond with JSON
w.Header().Set(contentTypeLabel, contentTypeJSON)
err = json.NewEncoder(w).Encode(results)
query, err := extractQueryOrBody(r)
if err != nil {
cmn.DebugMsg(cmn.DbgLvlDebug3, jsonErrorPrefix, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in removeSource request", http.StatusBadRequest)
return
}

results, err := performRemoveSource(query, getQType(r.Method != "POST"))
handleErrorAndRespond(w, err, results, "Error performing removeSource: %v", http.StatusInternalServerError)
}

0 comments on commit bd9a133

Please sign in to comment.