Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev df http config #270

Open
wants to merge 4 commits into
base: dev
Choose a base branch
from
Open

Dev df http config #270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b19fa17
Add support for loading configuration from a URL
fils Sep 19, 2024
19e4438
Add cfgURL flag and sanitize localConfig.yaml
fils Nov 15, 2024
135f5bb
Update config flag names and add new dependencies
fils Nov 15, 2024
a9ffbdd
Add support for loading config from a URL
fils Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 16 additions & 9 deletions cmd/gleaner/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,17 @@ import (
"flag"
"fmt"
"github.com/gleanerio/gleaner/internal/check"
"github.com/gleanerio/gleaner/internal/common"
"github.com/gleanerio/gleaner/internal/config"
"github.com/gleanerio/gleaner/internal/objects"
"github.com/gleanerio/gleaner/pkg"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"os"
"path/filepath"

"github.com/spf13/viper"

"github.com/gleanerio/gleaner/internal/common"
"github.com/gleanerio/gleaner/internal/objects"
)

var viperVal, sourceVal, modeVal, logVal string
var viperVal, viperURL, sourceVal, modeVal, logVal string
var setupVal, rudeVal bool

// pass -ldflags "-X main.version=testline"
Expand Down Expand Up @@ -49,6 +47,7 @@ func init() {
flag.StringVar(&sourceVal, "source", "", "Override config file source(s) to specify an index target")
flag.BoolVar(&rudeVal, "rude", false, "Ignore any robots.txt crawl delays or allow / disallow statements")
flag.StringVar(&viperVal, "cfg", "config", "Configuration file (can be YAML, JSON) Do NOT provide the extension in the command line. -cfg file not -cfg file.yml")
flag.StringVar(&viperURL, "cfgURL", "configurl", "Configuration URL (can be YAML, JSON)")
flag.StringVar(&modeVal, "mode", "full", "Set the mode (full | diff) to index all or just diffs")
flag.StringVar(&logVal, "log", "warn", "The log level to output (trace | debug | info | warn | error | fatal)")
}
Expand Down Expand Up @@ -88,13 +87,17 @@ func main() {

// Load the config file and set some defaults (config overrides)
if isFlagPassed("cfg") {
//v1, err = readConfig(viperVal, map[string]interface{}{})
v1, err = config.ReadGleanerConfig(filepath.Base(viperVal), filepath.Dir(viperVal))
if err != nil {
log.Fatal("error when reading config:", err)
}
} else if isFlagPassed("cfgURL") {
v1, err = config.ReadGleanerConfigURL(viperURL)
if err != nil {
log.Fatal("error when reading config:", err)
}
} else {
log.Error("Gleaner must be run with a config file: -cfg CONFIGFILE")
log.Error("Gleaner must be run with a config file or url: -cfg CONFIGFILE -cfgURL URL")
flag.Usage()
os.Exit(0)
}
Expand Down Expand Up @@ -189,7 +192,11 @@ func main() {
}()

//cli(mc, v1, db)
pkg.Cli(mc, v1) // move to a common call in batch.go
err = pkg.Cli(mc, v1)
if err != nil {
log.Error(err)
return
} // move to a common call in batch.go
}

func cleanUp() {
Expand Down
5 changes: 3 additions & 2 deletions configs/template/localConfig.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@ minio:
ssl: false
bucket: gleaner
region:
accessKey: worldsbestaccesskey
secretKey: worldsbestsecretkey
accessKey:
secretKey:
region:

# can be overridden with MINIO_BUCKET
sparql:
Expand Down
27 changes: 27 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,27 +31,42 @@ require (
)

require (
cloud.google.com/go v0.100.2 // indirect
cloud.google.com/go/compute v1.6.1 // indirect
cloud.google.com/go/firestore v1.6.1 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/apache/thrift v0.14.1 // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.3 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/gosuri/uilive v0.0.4 // indirect
github.com/hashicorp/consul/api v1.12.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-hclog v1.2.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/serf v0.9.7 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
Expand All @@ -60,20 +75,24 @@ require (
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/meilisearch/meilisearch-go v0.21.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/sagikazarmark/crypt v0.6.0 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
Expand All @@ -83,11 +102,19 @@ require (
github.com/tidwall/pretty v1.2.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d // indirect
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
go.etcd.io/etcd/client/v2 v2.305.4 // indirect
go.etcd.io/etcd/client/v3 v3.5.4 // indirect
go.opencensus.io v0.23.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.17.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect
google.golang.org/grpc v1.46.2 // indirect
Expand Down
Loading