From 80ab5a40de2ffb6ee7eaa047752a8f5c84971a1e Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Sun, 5 Sep 2021 14:53:10 -0700 Subject: [PATCH] feat: improve usage guide for run and lint command --- agent/agent.go | 6 +-- cmd/info.go | 46 +++++++++++++++----- cmd/lint.go | 34 +++++++++++++-- cmd/run.go | 45 ++++++++++++++++--- {example => docs/example}/README.md | 0 {example => docs/example}/date-kafka.yaml | 0 {example => docs/example}/kafka-console.yaml | 0 7 files changed, 106 insertions(+), 25 deletions(-) rename {example => docs/example}/README.md (100%) rename {example => docs/example}/date-kafka.yaml (100%) rename {example => docs/example}/kafka-console.yaml (100%) diff --git a/agent/agent.go b/agent/agent.go index 6222a3f48..154075193 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -37,7 +37,7 @@ func NewAgent(ef *registry.ExtractorFactory, pf *registry.ProcessorFactory, sf * func (r *Agent) Validate(rcp recipe.Recipe) (errs []error) { ext, err := r.extractorFactory.Get(rcp.Source.Type) if err != nil { - errs = append(errs, errors.Wrapf(err, "could not find %s (%s)", rcp.Source.Type, plugins.PluginTypeExtractor)) + errs = append(errs, errors.Wrapf(err, "invalid config for %s (%s)", rcp.Source.Type, plugins.PluginTypeExtractor)) } else { err = ext.Validate(rcp.Source.Config) if err != nil { @@ -48,7 +48,7 @@ func (r *Agent) Validate(rcp recipe.Recipe) (errs []error) { for _, s := range rcp.Sinks { sink, err := r.sinkFactory.Get(s.Name) if err != nil { - errs = append(errs, errors.Wrapf(err, "could not find %s (%s)", s.Name, plugins.PluginTypeSink)) + errs = append(errs, errors.Wrapf(err, "invalid config for %s (%s)", rcp.Source.Type, plugins.PluginTypeExtractor)) continue } err = sink.Validate(s.Config) @@ -60,7 +60,7 @@ func (r *Agent) Validate(rcp recipe.Recipe) (errs []error) { for _, p := range rcp.Processors { procc, err := r.processorFactory.Get(p.Name) if err != nil { - errs = append(errs, errors.Wrapf(err, "could not find %s (%s)", p.Name, plugins.PluginTypeProcessor)) + errs = append(errs, errors.Wrapf(err, "invalid config for %s (%s)", rcp.Source.Type, plugins.PluginTypeExtractor)) continue } err = procc.Validate(p.Config) diff --git a/cmd/info.go b/cmd/info.go index 51936d1fe..9bd56451a 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" + "github.com/MakeNowJust/heredoc" "github.com/charmbracelet/glamour" "github.com/odpf/meteor/registry" "github.com/odpf/salt/log" @@ -28,10 +29,17 @@ func InfoCmd(lg log.Logger) *cobra.Command { // InfoSinkCmd creates a command object for listing sinks func InfoSinkCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "sink ", - Example: "meteor info sink console", - Short: "Vew an sink information", - Args: cobra.ExactArgs(1), + Use: "sink ", + Short: "Vew sink information", + Long: heredoc.Doc(` + View sink information. + + The list of supported sinks is available via the 'meteor list sinks' command. + `), + Example: heredoc.Doc(` + $ meteor info sink console + `), + Args: cobra.ExactArgs(1), Annotations: map[string]string{ "group:core": "true", }, @@ -51,10 +59,17 @@ func InfoSinkCmd() *cobra.Command { // InfoExtCmd creates a command object for listing extractors func InfoExtCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "extractor ", - Example: "meteor info extractor kafka", - Short: "Vew extractor information", - Args: cobra.ExactArgs(1), + Use: "extractor ", + Short: "Vew extractor information", + Long: heredoc.Doc(` + View sink information. + + The list of supported extractors is available via the 'meteor list extractors' command. + `), + Example: heredoc.Doc(` + $ meteor info sink console + `), + Args: cobra.ExactArgs(1), Annotations: map[string]string{ "group:core": "true", }, @@ -73,10 +88,17 @@ func InfoExtCmd() *cobra.Command { // InfoProccCmd creates a command object for listing processors func InfoProccCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "processor ", - Example: "meteor info processor enrich", - Short: "Vew processor information", - Args: cobra.ExactArgs(1), + Use: "processor ", + Short: "Vew processor information", + Long: heredoc.Doc(` + View processor information. + + The list of supported processors is available via the 'meteor list processors' command. + `), + Example: heredoc.Doc(` + $ meteor info sink console + `), + Args: cobra.ExactArgs(1), Annotations: map[string]string{ "group:core": "true", }, diff --git a/cmd/lint.go b/cmd/lint.go index 23806ced5..ea8b1802b 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" + "github.com/MakeNowJust/heredoc" "github.com/odpf/meteor/agent" "github.com/odpf/meteor/metrics" "github.com/odpf/meteor/recipe" @@ -19,8 +20,22 @@ func LintCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command { Use: "lint [path]", Aliases: []string{"l"}, Args: cobra.ExactValidArgs(1), - Example: "meteor lint recipe.yaml", - Short: "Validate recipes.", + Short: "Validate recipes", + Long: heredoc.Doc(` + Validate specified recipes. + + Linters are run on the recipe files in the specified path. + If no path is specified, the current directory is used. + `), + Example: heredoc.Doc(` + $ meteor lint recipe.yml + + # lint all recipes in the specified directory + $ meteor lint _recipes/ + + # lint all recipes in the current directory + $ meteor lint . + `), Annotations: map[string]string{ "group:core": "true", }, @@ -36,20 +51,31 @@ func LintCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command { if len(recipes) == 0 { fmt.Println(cs.Yellowf("no recipe found in [%s]", args[0])) + fmt.Println(cs.Blue("\nUse 'meteor gen recipe' to generate a new recipe.")) return nil } + report := []string{""} + var success = 0 + var failures = 0 for _, recipe := range recipes { errs := runner.Validate(recipe) if len(errs) > 0 { - // Print errors for _, err := range errs { lg.Error(err.Error()) } + report = append(report, fmt.Sprint(cs.FailureIcon(), cs.Redf(" %d errors found is recipe %s", len(errs), recipe.Name))) + failures++ continue } - fmt.Println(cs.Greenf("recipe [%s] is valid", recipe.Name)) + report = append(report, fmt.Sprint(cs.SuccessIcon(), cs.Greenf(" 0 error found in recipe %s", recipe.Name))) + success++ + } + + for _, line := range report { + fmt.Println(line) } + fmt.Printf("Total: %d, Success: %d, Failures: %d\n", len(recipes), success, failures) return nil }, } diff --git a/cmd/run.go b/cmd/run.go index f58ded7f3..70673d086 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" + "github.com/MakeNowJust/heredoc" "github.com/odpf/meteor/agent" "github.com/odpf/meteor/metrics" "github.com/odpf/meteor/recipe" @@ -15,10 +16,29 @@ import ( // RunCmd creates a command object for the "run" action. func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command { return &cobra.Command{ - Use: "run [COMMAND]", - Example: "meteor run recipe.yaml", - Short: "Run meteor for provided recipes.", - Args: cobra.ExactArgs(1), + Use: "run [COMMAND]", + Short: "Run meteor for specified recipes.", + Long: heredoc.Doc(` + Run meteor for specified recipes. + + A recipe is a set of instructions and configurations defined by user, + and in Meteor they are used to define how metadata will be collected. + + If a recipe file is provided, recipe will be + executed as a single recipe. + If a recipe directory is provided, recipes will + be executed as a group of recipes. + `), + Example: heredoc.Doc(` + $ meteor run recipe.yml + + # run all recipes in the specified directory + $ meteor run _recipes/ + + # run all recipes in the current directory + $ meteor lint . + `), + Args: cobra.ExactArgs(1), Annotations: map[string]string{ "group:core": "true", }, @@ -33,19 +53,32 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command { } if len(recipes) == 0 { - fmt.Println(cs.Yellowf("no recipe found in [%s]", args[0])) + fmt.Println(cs.WarningIcon(), cs.Yellowf(" no recipe found in [%s]", args[0])) return nil } + report := []string{""} + var success = 0 + var failures = 0 + runs := runner.RunMultiple(recipes) for _, run := range runs { lg.Debug("recipe details", "recipe", run.Recipe) if run.Error != nil { lg.Error(run.Error.Error(), "recipe", run.Recipe.Name) + report = append(report, fmt.Sprint(cs.FailureIcon(), cs.Redf(" failed to run recipe %s", run.Recipe.Name))) + failures++ continue } - lg.Info("success", "recipe", run.Recipe.Name) + success++ + report = append(report, fmt.Sprint(cs.SuccessIcon(), cs.Greenf(" successfully ran recipe `%s`", run.Recipe.Name))) } + + for _, line := range report { + fmt.Println(line) + } + fmt.Printf("Total: %d, Success: %d, Failures: %d\n", len(recipes), success, failures) + return nil }, } diff --git a/example/README.md b/docs/example/README.md similarity index 100% rename from example/README.md rename to docs/example/README.md diff --git a/example/date-kafka.yaml b/docs/example/date-kafka.yaml similarity index 100% rename from example/date-kafka.yaml rename to docs/example/date-kafka.yaml diff --git a/example/kafka-console.yaml b/docs/example/kafka-console.yaml similarity index 100% rename from example/kafka-console.yaml rename to docs/example/kafka-console.yaml