From 9442a9b1135841b6f0ee47d66ca2fb0288d267e0 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Thu, 27 Jun 2024 19:43:48 +0330 Subject: [PATCH 01/19] feat: sanitizer --- cmd/parser/config.go | 8 +++++++ cmd/parser/cron_parser.go | 46 +++++++++++++++++++++++++++++++++++++++ cmd/parser/parser.go | 31 ++++++++++++++++++++++++++ cmd/root.go | 11 +++++++--- 4 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 cmd/parser/config.go create mode 100644 cmd/parser/cron_parser.go create mode 100644 cmd/parser/parser.go diff --git a/cmd/parser/config.go b/cmd/parser/config.go new file mode 100644 index 0000000..a03bcdf --- /dev/null +++ b/cmd/parser/config.go @@ -0,0 +1,8 @@ +package parser + +type parserConfig struct { + cronFile string + output string + matcher string + hasUser bool +} diff --git a/cmd/parser/cron_parser.go b/cmd/parser/cron_parser.go new file mode 100644 index 0000000..632ba2a --- /dev/null +++ b/cmd/parser/cron_parser.go @@ -0,0 +1,46 @@ +package parser + +import ( + "fmt" + "log" + "os" + "regexp" + + "github.com/FMotalleb/crontab-go/config" +) + +type cronString struct { + string +} + +func (c cronString) sanitizeLineBreaker() cronString { + reg, _ := regexp.Compile(`\s*\\\s*[\r\n|\n]\s*([\r\n|\n\s])*`) + out := reg.ReplaceAllString(c.string, " ") + return cronString{out} +} + +func (c cronString) sanitizeComments() cronString { + reg, _ := regexp.Compile(`\s*#.*`) + out := reg.ReplaceAllString(c.string, "") + return cronString{out} +} + +func (cfg parserConfig) parse() config.Config { + file, err := os.OpenFile(cfg.cronFile, os.O_RDONLY, 0o644) + if err != nil { + log.Panicf("can't open cron file: %v", err) + } + stat, err := file.Stat() + content := make([]byte, stat.Size()) + file.Read(content) + strContent := cronString{string(content)} + strContent = strContent. + sanitizeComments() + fmt.Println( + strContent.sanitizeComments(), + strContent.sanitizeComments().sanitizeLineBreaker().sanitizeLineBreaker(), + ) + + c := config.Config{} + return c +} diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go new file mode 100644 index 0000000..c913351 --- /dev/null +++ b/cmd/parser/parser.go @@ -0,0 +1,31 @@ +package parser + +import ( + "errors" + "log" + + "github.com/spf13/cobra" +) + +var ( + cfg = &parserConfig{} + ParserCmd = &cobra.Command{ + Use: "parse ", + ValidArgs: []string{"crontab file path"}, + Short: "Parse crontab syntax and converts it into yaml syntax for crontab-go", + Run: func(cmd *cobra.Command, args []string) { + cfg.cronFile = cmd.Flags().Arg(0) + log.SetPrefix("[cron-parser]") + if cfg.cronFile == "" { + log.Panicln(errors.New("no crontab file specified, see usage using --help flag")) + } + cfg.parse() + }, + } +) + +func init() { + ParserCmd.PersistentFlags().StringVarP(&cfg.output, "output", "o", "", "output file to write configuration to") + ParserCmd.PersistentFlags().BoolVarP(&cfg.hasUser, "with-user", "u", false, "indicates that whether the given cron file has user field") + ParserCmd.PersistentFlags().StringVar(&cfg.matcher, "matcher", `(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\d+(ns|us|µs|ms|s|m|h))+)|((((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|\*) ?){5,7})`, "matcher for cron") +} diff --git a/cmd/root.go b/cmd/root.go index 7e8bcfa..e5e610c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" + "github.com/FMotalleb/crontab-go/cmd/parser" "github.com/FMotalleb/crontab-go/config" ) @@ -27,7 +28,10 @@ designed to replace the traditional crontab in Docker environments. With its seamless integration and easy-to-use YAML configuration, Cronjob-go simplifies the process of scheduling and managing recurring tasks within your containerized applications.`, - Run: func(cmd *cobra.Command, args []string) {}, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(cmd.Args) + initConfig() + }, } func Execute() { @@ -39,9 +43,11 @@ func Execute() { func init() { warnOnErr(godotenv.Load(), "Cannot initialize .env file: %s") - cobra.OnInitialize(initConfig) + rootCmd.AddCommand(parser.ParserCmd) rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is config.yaml)") + + // cobra.OnInitialize() } func warnOnErr(err error, message string) { @@ -161,7 +167,6 @@ func initConfig() { } viper.AutomaticEnv() - panicOnErr( viper.ReadInConfig(), "Cannot read the config file: %s", From 9fba9c006a211adbc52ef6b2209ae1578d1e425b Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 17:26:30 +0330 Subject: [PATCH 02/19] feat: fully functional cron parser --- cmd/parser/config.go | 8 +- cmd/parser/cron_parser.go | 41 +-------- cmd/parser/cron_spec.go | 50 +++++++++++ cmd/parser/cron_string.go | 176 ++++++++++++++++++++++++++++++++++++++ cmd/parser/parser.go | 33 ++++++- 5 files changed, 264 insertions(+), 44 deletions(-) create mode 100644 cmd/parser/cron_spec.go create mode 100644 cmd/parser/cron_string.go diff --git a/cmd/parser/config.go b/cmd/parser/config.go index a03bcdf..8148cad 100644 --- a/cmd/parser/config.go +++ b/cmd/parser/config.go @@ -1,8 +1,8 @@ package parser type parserConfig struct { - cronFile string - output string - matcher string - hasUser bool + cronFile string + output string + cronMatcher string + hasUser bool } diff --git a/cmd/parser/cron_parser.go b/cmd/parser/cron_parser.go index 632ba2a..f0f1fd8 100644 --- a/cmd/parser/cron_parser.go +++ b/cmd/parser/cron_parser.go @@ -1,46 +1,13 @@ package parser import ( - "fmt" - "log" - "os" - "regexp" - "github.com/FMotalleb/crontab-go/config" ) -type cronString struct { - string -} - -func (c cronString) sanitizeLineBreaker() cronString { - reg, _ := regexp.Compile(`\s*\\\s*[\r\n|\n]\s*([\r\n|\n\s])*`) - out := reg.ReplaceAllString(c.string, " ") - return cronString{out} -} - -func (c cronString) sanitizeComments() cronString { - reg, _ := regexp.Compile(`\s*#.*`) - out := reg.ReplaceAllString(c.string, "") - return cronString{out} -} - func (cfg parserConfig) parse() config.Config { - file, err := os.OpenFile(cfg.cronFile, os.O_RDONLY, 0o644) - if err != nil { - log.Panicf("can't open cron file: %v", err) - } - stat, err := file.Stat() - content := make([]byte, stat.Size()) - file.Read(content) - strContent := cronString{string(content)} - strContent = strContent. - sanitizeComments() - fmt.Println( - strContent.sanitizeComments(), - strContent.sanitizeComments().sanitizeLineBreaker().sanitizeLineBreaker(), + cron := NewCronFromFile(cfg.cronFile) + return *cron.ParseConfig( + cfg.cronMatcher, + cfg.hasUser, ) - - c := config.Config{} - return c } diff --git a/cmd/parser/cron_spec.go b/cmd/parser/cron_spec.go new file mode 100644 index 0000000..292e66c --- /dev/null +++ b/cmd/parser/cron_spec.go @@ -0,0 +1,50 @@ +package parser + +import ( + "log" + "regexp" +) + +type ( + cronSpecParser = func([]string, map[string]string) *cronSpec + cronSpec struct { + timing string + user string + command string + environ map[string]string + } +) + +func normalParser(regex *regexp.Regexp) cronSpecParser { + cronIndex := regex.SubexpIndex("cron") + // userIndex := regex.SubexpIndex("user") + cmdIndex := regex.SubexpIndex("cmd") + if cronIndex < 0 || cmdIndex < 0 { + log.Panicf("cannot find groups (cron,cmd) in regexp: `%v`", regex.SubexpIndex) + } + return func(match []string, env map[string]string) *cronSpec { + return &cronSpec{ + timing: match[cronIndex], + user: "", + command: match[cmdIndex], + environ: env, + } + } +} + +func withUserParser(regex *regexp.Regexp) cronSpecParser { + cronIndex := regex.SubexpIndex("cron") + userIndex := regex.SubexpIndex("user") + cmdIndex := regex.SubexpIndex("cmd") + if cronIndex < 0 || cmdIndex < 0 || userIndex < 0 { + log.Panicf("cannot find groups (cron,user,cmd) in regexp: `%v`", regex.SubexpIndex) + } + return func(match []string, env map[string]string) *cronSpec { + return &cronSpec{ + timing: match[cronIndex], + user: match[userIndex], + command: match[cmdIndex], + environ: env, + } + } +} diff --git a/cmd/parser/cron_string.go b/cmd/parser/cron_string.go new file mode 100644 index 0000000..dab8d7b --- /dev/null +++ b/cmd/parser/cron_string.go @@ -0,0 +1,176 @@ +package parser + +import ( + "fmt" + "log" + "os" + "regexp" + "strings" + + "github.com/FMotalleb/crontab-go/config" +) + +var envRegex = regexp.MustCompile(`^(?[\w\d_]+)=(?.*)$`) + +type CronString struct { + string +} + +func NewCronString(cron string) CronString { + return CronString{cron} +} + +func NewCronFromFile(filePath string) CronString { + file, err := os.OpenFile(filePath, os.O_RDONLY, 0o644) + if err != nil { + log.Panicf("can't open cron file: %v", err) + } + stat, err := file.Stat() + content := make([]byte, stat.Size()) + file.Read(content) + return CronString{string(content)} +} + +func (s CronString) replaceAll(regex string, repl string) CronString { + reg := regexp.MustCompile(regex) + out := reg.ReplaceAllString(s.string, repl) + return CronString{out} +} + +func (s CronString) sanitizeLineBreaker() CronString { + return s.replaceAll( + `\s*\\\s*\n\s*([\n|\n\s])*`, + " ", + ) +} + +func (s CronString) sanitizeEmptyLine() CronString { + return s.replaceAll( + `\n\s*\n`, + "\n", + ) +} + +func (s CronString) sanitizeComments() CronString { + return s.replaceAll( + `\s*#.*`, + "", + ) +} + +func (s CronString) sanitize() CronString { + return s. + replaceAll("\r\n", "\n"). + sanitizeComments(). + sanitizeLineBreaker(). + sanitizeEmptyLine() +} + +func (s CronString) lines() []string { + return strings.Split(s.string, "\n") +} + +type cronLine struct { + string +} + +func (l cronLine) exportEnv() map[string]string { + match := envRegex.FindStringSubmatch(l.string) + answer := make(map[string]string) + switch len(match) { + case 0: + case 3: + answer[match[1]] = match[2] + default: + log.Panicf("found multiple(%d) env vars in single line\n please attach your crontab file too\n affected line: %s\n parser result: %#v\n", len(match), l.string, match) + } + return answer +} + +func (l cronLine) exportSpec(regex *regexp.Regexp, env map[string]string, parser cronSpecParser) *cronSpec { + match := regex.FindStringSubmatch(l.string) + if len(match) < 1 { + return nil + } + return parser(match, env) +} + +func (c *CronString) parseAsSpec( + pattern string, + hasUser bool, +) []cronSpec { + envTable := make(map[string]string) + specs := make([]cronSpec, 0) + lines := c.sanitize().lines() + lineParser := "(?.*)" + if hasUser { + lineParser = fmt.Sprintf(`(?\w[\w\d]*)\s+%s`, lineParser) + } + cronLineMatcher := fmt.Sprintf(`^(?%s)\s+%s$`, pattern, lineParser) + + matcher, err := regexp.Compile(cronLineMatcher) + if err != nil { + log.Panicf("cannot parse cron `%s`", matcher) + } + var parser cronSpecParser + if hasUser { + parser = withUserParser(matcher) + } else { + parser = normalParser(matcher) + } + + for _, line := range lines { + l := cronLine{line} + for key, val := range l.exportEnv() { + if old, ok := envTable[key]; ok { + log.Printf("env var of key `%s`, value `%s`, is going to be replaced by `%s`\n", key, old, val) + } + envTable[key] = val + } + if spec := l.exportSpec(matcher, envTable, parser); spec != nil { + specs = append(specs, *spec) + } + + } + return specs +} + +func (c *CronString) ParseConfig( + pattern string, + hasUser bool, +) *config.Config { + specs := c.parseAsSpec(pattern, hasUser) + cfg := &config.Config{} + for _, spec := range specs { + addSpec(cfg, spec) + } + return cfg +} + +func addSpec(cfg *config.Config, spec cronSpec) { + jobName := fmt.Sprintf("FromCron: %s", spec.timing) + + for _, job := range cfg.Jobs { + if job.Name == jobName { + job.Tasks = append( + job.Tasks, + config.Task{ + Command: spec.command, + Env: spec.environ, + }, + ) + return + } + } + job := &config.JobConfig{} + job.Name = jobName + job.Description = "Imported from cron file" + job.Disabled = false + job.Events = []config.JobEvent{ + { + Cron: spec.timing, + }, + } + cfg.Jobs = append(cfg.Jobs, job) + addSpec(cfg, spec) +} diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go index c913351..c6a47ff 100644 --- a/cmd/parser/parser.go +++ b/cmd/parser/parser.go @@ -1,10 +1,15 @@ package parser import ( + "bytes" + "encoding/json" "errors" + "io" "log" + "os" "github.com/spf13/cobra" + "gopkg.in/yaml.v3" ) var ( @@ -15,11 +20,33 @@ var ( Short: "Parse crontab syntax and converts it into yaml syntax for crontab-go", Run: func(cmd *cobra.Command, args []string) { cfg.cronFile = cmd.Flags().Arg(0) - log.SetPrefix("[cron-parser]") + log.SetPrefix("[Cron Parser]") if cfg.cronFile == "" { log.Panicln(errors.New("no crontab file specified, see usage using --help flag")) } - cfg.parse() + finalConfig := cfg.parse() + str, err := json.Marshal(finalConfig) + if err != nil { + log.Panicf("failed to marshal final config: %v", err) + } + hashMap := make(map[string]any) + json.Unmarshal(str, &hashMap) + ans, _ := yaml.Marshal(hashMap) + result := string(ans) + log.Printf("output:\n%s", result) + if cfg.output != "" { + outputFile, err := os.OpenFile(cfg.output, os.O_WRONLY|os.O_CREATE, 0o644) + if err != nil { + log.Panicf("failed to open output file: %v", err) + } + buf := bytes.NewBufferString(result) + _, err = io.Copy(outputFile, buf) + if err != nil { + log.Panicf("failed to write output file: %v", err) + } + } + log.Println("Done writing output") + os.Exit(0) }, } ) @@ -27,5 +54,5 @@ var ( func init() { ParserCmd.PersistentFlags().StringVarP(&cfg.output, "output", "o", "", "output file to write configuration to") ParserCmd.PersistentFlags().BoolVarP(&cfg.hasUser, "with-user", "u", false, "indicates that whether the given cron file has user field") - ParserCmd.PersistentFlags().StringVar(&cfg.matcher, "matcher", `(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\d+(ns|us|µs|ms|s|m|h))+)|((((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|\*) ?){5,7})`, "matcher for cron") + ParserCmd.PersistentFlags().StringVar(&cfg.cronMatcher, "matcher", `(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\d+(ns|us|µs|ms|s|m|h))+)|((((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|\*|(\*\/\d)) ?){5,7})`, "matcher for cron") } From 93188f4358a7de8f94c39bcc519eb8e43f2eece8 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 17:27:04 +0330 Subject: [PATCH 03/19] feat: integrated cron parser --- .vscode/launch.json | 15 ++++++++++++++- config/config.go | 22 +++++++++++----------- config/config_validtor_test.go | 12 ++++++------ core/jobs/runner.go | 6 +++--- main.go | 17 ++++++++++++++++- meta/github.go | 11 +++++++++++ 6 files changed, 61 insertions(+), 22 deletions(-) create mode 100644 meta/github.go diff --git a/.vscode/launch.json b/.vscode/launch.json index ad1d727..1cbe3fc 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -15,6 +15,19 @@ "-c", "config.local.yaml" ] - } + }, + { + "name": "Parse", + "type": "go", + "request": "launch", + "mode": "auto", + "program": ".", + "env": {}, + "args": [ + "parse", + "crontab.log", + "-u" + ] + }, ] } \ No newline at end of file diff --git a/config/config.go b/config/config.go index 9d5f9d0..74c671e 100644 --- a/config/config.go +++ b/config/config.go @@ -10,23 +10,23 @@ import ( // Config represents the configuration for the crontab application. type Config struct { // Log configs - LogTimestampFormat string `mapstructure:"log_timestamp_format" json:"log_timestamp_format"` - LogFormat enums.LoggerFormatType `mapstructure:"log_format" json:"log_format"` + LogTimestampFormat string `mapstructure:"log_timestamp_format" json:"log_timestamp_format,omitempty"` + LogFormat enums.LoggerFormatType `mapstructure:"log_format" json:"log_format,omitempty"` LogFile string `mapstructure:"log_file" json:"log_file,omitempty"` - LogStdout bool `mapstructure:"log_stdout" json:"log_stdout"` - LogLevel enums.LogLevel `mapstructure:"log_level" json:"log_level"` + LogStdout bool `mapstructure:"log_stdout" json:"log_stdout,omitempty"` + LogLevel enums.LogLevel `mapstructure:"log_level" json:"log_level,omitempty"` // Command executor configs - Shell string `mapstructure:"shell" json:"shell"` - ShellArgs []string `mapstructure:"shell_args" json:"shell_args"` + Shell string `mapstructure:"shell" json:"shell,omitempty"` + ShellArgs []string `mapstructure:"shell_args" json:"shell_args,omitempty"` // Web-server config - WebServerAddress string `mapstructure:"webserver_address" json:"webserver_listen_address"` - WebServerPort uint `mapstructure:"webserver_port" json:"webserver_port"` - WebserverUsername string `mapstructure:"webserver_username" json:"webserver_username"` - WebServerPassword string `mapstructure:"webserver_password" json:"webserver_password"` + WebServerAddress string `mapstructure:"webserver_address" json:"webserver_listen_address,omitempty"` + WebServerPort uint `mapstructure:"webserver_port" json:"webserver_port,omitempty"` + WebserverUsername string `mapstructure:"webserver_username" json:"webserver_username,omitempty"` + WebServerPassword string `mapstructure:"webserver_password" json:"webserver_password,omitempty"` - Jobs []JobConfig `mapstructure:"jobs" json:"jobs"` + Jobs []*JobConfig `mapstructure:"jobs" json:"jobs"` } // JobConfig represents the configuration for a specific job. diff --git a/config/config_validtor_test.go b/config/config_validtor_test.go index 36d785a..f3bb99d 100644 --- a/config/config_validtor_test.go +++ b/config/config_validtor_test.go @@ -31,7 +31,7 @@ func TestConfig_Validate_LogFormatFails(t *testing.T) { cfg := &config.Config{ LogFormat: enums.LoggerFormatType("unknown"), LogLevel: enums.DebugLevel, - Jobs: []config.JobConfig{}, + Jobs: []*config.JobConfig{}, } log, _ := mocklogger.HijackOutput(logrus.New()) err := cfg.Validate(log.WithField("test", "test")) @@ -43,7 +43,7 @@ func TestConfig_Validate_LogLevelFails(t *testing.T) { cfg := &config.Config{ LogFormat: enums.AnsiLogger, LogLevel: enums.LogLevel("unknown"), - Jobs: []config.JobConfig{}, + Jobs: []*config.JobConfig{}, } logger, _ := mocklogger.HijackOutput(logrus.New()) log := logrus.NewEntry(logger) @@ -56,7 +56,7 @@ func TestConfig_Validate_JobFails(t *testing.T) { cfg := &config.Config{ LogFormat: enums.JSONLogger, LogLevel: enums.FatalLevel, - Jobs: []config.JobConfig{failJob}, + Jobs: []*config.JobConfig{&failJob}, } logger, _ := mocklogger.HijackOutput(logrus.New()) log := logrus.NewEntry(logger) @@ -68,8 +68,8 @@ func TestConfig_Validate_AllValidationsPass(t *testing.T) { cfg := &config.Config{ LogFormat: enums.JSONLogger, LogLevel: enums.DebugLevel, - Jobs: []config.JobConfig{ - okJob, + Jobs: []*config.JobConfig{ + &okJob, }, } logger, _ := mocklogger.HijackOutput(logrus.New()) @@ -82,7 +82,7 @@ func TestConfig_Validate_NoJobs(t *testing.T) { cfg := &config.Config{ LogFormat: enums.JSONLogger, LogLevel: enums.DebugLevel, - Jobs: []config.JobConfig{}, + Jobs: []*config.JobConfig{}, } logger, _ := mocklogger.HijackOutput(logrus.New()) log := logrus.NewEntry(logger) diff --git a/core/jobs/runner.go b/core/jobs/runner.go index 4c5eacc..702d43a 100644 --- a/core/jobs/runner.go +++ b/core/jobs/runner.go @@ -34,9 +34,9 @@ func InitializeJobs(log *logrus.Entry, cronInstance *cron.Cron) { log.Panicf("failed to validate job (%s): %v", job.Name, err) } - signal := buildSignal(job, cronInstance, logger) + signal := buildSignal(*job, cronInstance, logger) - tasks, doneHooks, failHooks := initTasks(job, logger) + tasks, doneHooks, failHooks := initTasks(*job, logger) logger.Trace("Tasks initialized") go taskHandler(c, logger, signal, tasks, doneHooks, failHooks, lock) @@ -54,7 +54,7 @@ func buildSignal(job config.JobConfig, cronInstance *cron.Cron, logger *logrus.E return signal } -func initLogger(c context.Context, log *logrus.Entry, job config.JobConfig) *logrus.Entry { +func initLogger(c context.Context, log *logrus.Entry, job *config.JobConfig) *logrus.Entry { logger := log.WithContext(c).WithField("job.name", job.Name) logger.Trace("Initializing Job") return logger diff --git a/main.go b/main.go index 13fbed0..6ce87e3 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,9 @@ along with this program. If not, see . package main import ( + "log" + "os" + "github.com/robfig/cron/v3" "github.com/FMotalleb/crontab-go/cmd" @@ -24,13 +27,25 @@ import ( "github.com/FMotalleb/crontab-go/core/jobs" "github.com/FMotalleb/crontab-go/core/webserver" "github.com/FMotalleb/crontab-go/logger" + "github.com/FMotalleb/crontab-go/meta" ) func main() { + defer func() { + log.Printf("here") + if err := recover(); err != nil { + log.Printf( + "exception: %v\n\nif you think this is an error from application please report at: %s", + err, + meta.Issues(), + ) + + os.Exit(1) + } + }() cmd.Execute() logger.InitFromConfig() log := logger.SetupLogger("Crontab-GO") - // TODO: move somewhere else cronInstance := cron.New(cron.WithSeconds()) log.Info("Booting up") jobs.InitializeJobs(log, cronInstance) diff --git a/meta/github.go b/meta/github.go new file mode 100644 index 0000000..e02d47c --- /dev/null +++ b/meta/github.go @@ -0,0 +1,11 @@ +package meta + +import "fmt" + +func Project() string { + return "https://github.com/FMotalleb/crontab-go" +} + +func Issues() string { + return fmt.Sprintf("%s/issues", Project()) +} From 125e9a2315e4775acc13c767b896ae6a0206d882 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 17:39:00 +0330 Subject: [PATCH 04/19] fix: minor issues --- cmd/parser/cron_line.go | 36 ++++++++++++++ cmd/parser/cron_spec.go | 4 +- cmd/parser/cron_string.go | 100 +++++++++++++++++--------------------- main.go | 4 +- test.yaml | 49 +++++++++++++++++++ 5 files changed, 132 insertions(+), 61 deletions(-) create mode 100644 cmd/parser/cron_line.go create mode 100644 test.yaml diff --git a/cmd/parser/cron_line.go b/cmd/parser/cron_line.go new file mode 100644 index 0000000..d61105d --- /dev/null +++ b/cmd/parser/cron_line.go @@ -0,0 +1,36 @@ +package parser + +import ( + "log" + "regexp" + "strings" +) + +type cronLine struct { + string +} + +func (l cronLine) exportEnv() map[string]string { + match := envRegex.FindStringSubmatch(l.string) + answer := make(map[string]string) + switch len(match) { + case 0: + case 3: + answer[match[1]] = match[2] + default: + log.Panicf("found multiple(%d) env vars in single line\n please attach your crontab file too\n affected line: %s\n parser result: %#v\n", len(match), l.string, match) + } + return answer +} + +func (l cronLine) exportSpec(regex *regexp.Regexp, env map[string]string, parser cronSpecParser) *cronSpec { + match := regex.FindStringSubmatch(l.string) + if len(match) < 1 { + if len(strings.Trim(l.string, " \n\t")) == 0 { + return nil + } else { + log.Panicf("cannot parse this non-empty line as cron specification: %s", l.string) + } + } + return parser(match, env) +} diff --git a/cmd/parser/cron_spec.go b/cmd/parser/cron_spec.go index 292e66c..a8932d7 100644 --- a/cmd/parser/cron_spec.go +++ b/cmd/parser/cron_spec.go @@ -20,7 +20,7 @@ func normalParser(regex *regexp.Regexp) cronSpecParser { // userIndex := regex.SubexpIndex("user") cmdIndex := regex.SubexpIndex("cmd") if cronIndex < 0 || cmdIndex < 0 { - log.Panicf("cannot find groups (cron,cmd) in regexp: `%v`", regex.SubexpIndex) + log.Panicf("cannot find groups (cron,cmd) in regexp: `%s", regex) } return func(match []string, env map[string]string) *cronSpec { return &cronSpec{ @@ -37,7 +37,7 @@ func withUserParser(regex *regexp.Regexp) cronSpecParser { userIndex := regex.SubexpIndex("user") cmdIndex := regex.SubexpIndex("cmd") if cronIndex < 0 || cmdIndex < 0 || userIndex < 0 { - log.Panicf("cannot find groups (cron,user,cmd) in regexp: `%v`", regex.SubexpIndex) + log.Panicf("cannot find groups (cron,user,cmd) in regexp: `%s", regex) } return func(match []string, env map[string]string) *cronSpec { return &cronSpec{ diff --git a/cmd/parser/cron_string.go b/cmd/parser/cron_string.go index dab8d7b..d1d7a6f 100644 --- a/cmd/parser/cron_string.go +++ b/cmd/parser/cron_string.go @@ -70,31 +70,6 @@ func (s CronString) lines() []string { return strings.Split(s.string, "\n") } -type cronLine struct { - string -} - -func (l cronLine) exportEnv() map[string]string { - match := envRegex.FindStringSubmatch(l.string) - answer := make(map[string]string) - switch len(match) { - case 0: - case 3: - answer[match[1]] = match[2] - default: - log.Panicf("found multiple(%d) env vars in single line\n please attach your crontab file too\n affected line: %s\n parser result: %#v\n", len(match), l.string, match) - } - return answer -} - -func (l cronLine) exportSpec(regex *regexp.Regexp, env map[string]string, parser cronSpecParser) *cronSpec { - match := regex.FindStringSubmatch(l.string) - if len(match) < 1 { - return nil - } - return parser(match, env) -} - func (c *CronString) parseAsSpec( pattern string, hasUser bool, @@ -102,35 +77,23 @@ func (c *CronString) parseAsSpec( envTable := make(map[string]string) specs := make([]cronSpec, 0) lines := c.sanitize().lines() - lineParser := "(?.*)" - if hasUser { - lineParser = fmt.Sprintf(`(?\w[\w\d]*)\s+%s`, lineParser) - } - cronLineMatcher := fmt.Sprintf(`^(?%s)\s+%s$`, pattern, lineParser) - - matcher, err := regexp.Compile(cronLineMatcher) - if err != nil { - log.Panicf("cannot parse cron `%s`", matcher) - } - var parser cronSpecParser - if hasUser { - parser = withUserParser(matcher) - } else { - parser = normalParser(matcher) - } + matcher, parser := buildMapper(hasUser, pattern) for _, line := range lines { l := cronLine{line} - for key, val := range l.exportEnv() { - if old, ok := envTable[key]; ok { - log.Printf("env var of key `%s`, value `%s`, is going to be replaced by `%s`\n", key, old, val) + if env := l.exportEnv(); len(env) > 0 { + for key, val := range l.exportEnv() { + if old, ok := envTable[key]; ok { + log.Printf("env var of key `%s`, value `%s`, is going to be replaced by `%s`\n", key, old, val) + } + envTable[key] = val + + } + } else { + if spec := l.exportSpec(matcher, envTable, parser); spec != nil { + specs = append(specs, *spec) } - envTable[key] = val - } - if spec := l.exportSpec(matcher, envTable, parser); spec != nil { - specs = append(specs, *spec) } - } return specs } @@ -147,30 +110,55 @@ func (c *CronString) ParseConfig( return cfg } +func buildMapper(hasUser bool, pattern string) (*regexp.Regexp, func([]string, map[string]string) *cronSpec) { + lineParser := "(?.*)" + if hasUser { + lineParser = fmt.Sprintf(`(?\w[\w\d]*)\s+%s`, lineParser) + } + cronLineMatcher := fmt.Sprintf(`^(?%s)\s+%s$`, pattern, lineParser) + + matcher, err := regexp.Compile(cronLineMatcher) + if err != nil { + log.Panicf("cannot parse cron `%s`", matcher) + } + var parser cronSpecParser + if hasUser { + parser = withUserParser(matcher) + } else { + parser = normalParser(matcher) + } + return matcher, parser +} + func addSpec(cfg *config.Config, spec cronSpec) { jobName := fmt.Sprintf("FromCron: %s", spec.timing) - for _, job := range cfg.Jobs { if job.Name == jobName { + task := config.Task{ + Command: spec.command, + UserName: spec.user, + Env: spec.environ, + } job.Tasks = append( job.Tasks, - config.Task{ - Command: spec.command, - Env: spec.environ, - }, + task, ) return } } + initJob(jobName, spec.timing, cfg) + addSpec(cfg, spec) +} + +func initJob(jobName string, timing string, cfg *config.Config) { job := &config.JobConfig{} job.Name = jobName job.Description = "Imported from cron file" job.Disabled = false job.Events = []config.JobEvent{ { - Cron: spec.timing, + Cron: timing, }, } cfg.Jobs = append(cfg.Jobs, job) - addSpec(cfg, spec) } diff --git a/main.go b/main.go index 6ce87e3..bd0cb17 100644 --- a/main.go +++ b/main.go @@ -32,14 +32,12 @@ import ( func main() { defer func() { - log.Printf("here") if err := recover(); err != nil { log.Printf( - "exception: %v\n\nif you think this is an error from application please report at: %s", + "recovering from a panic:\n%v\nif you think this is an error from application please report at: %s", err, meta.Issues(), ) - os.Exit(1) } }() diff --git a/test.yaml b/test.yaml new file mode 100644 index 0000000..ae55495 --- /dev/null +++ b/test.yaml @@ -0,0 +1,49 @@ +jobs: + - description: Imported from cron file + events: + - cron: '* * * * *' + hooks: {} + name: 'FromCron: * * * * *' + tasks: + - command: php -v --help + env: + PATH: C:\xampp\php; + user: tng + - command: php -v --help --params + env: + PATH: C:\xampp\php; + user: tng + - description: Imported from cron file + events: + - cron: 0 0 * * * + hooks: {} + name: 'FromCron: 0 0 * * *' + tasks: + - command: php -v --help --params + env: + PATH: C:\xampp\php; + user: tng + - command: php -v --help --params + env: + PATH: C:\xampp\php; + user: tng + - description: Imported from cron file + events: + - cron: 0 0 */2 * * + hooks: {} + name: 'FromCron: 0 0 */2 * *' + tasks: + - command: php -v --help --params + env: + PATH: C:\xampp\php; + user: tng +escription: Imported from cron file + events: + - cron: 0 0 */2 * * + hooks: {} + name: 'FromCron: 0 0 */2 * *' + tasks: + - command: php -v --help --params + env: + PATH: C:\xampp\php; + user: tng From 2b113190e80f7aa6c70ea0158a01d7628b7e94e7 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 17:41:45 +0330 Subject: [PATCH 05/19] fix: errors --- config/events_validator_test.go | 2 +- config/task_validator_test.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/config/events_validator_test.go b/config/events_validator_test.go index 3f333bb..c5c1136 100644 --- a/config/events_validator_test.go +++ b/config/events_validator_test.go @@ -81,7 +81,7 @@ func TestJobEvent_Validate_MultipleActiveSchedules(t *testing.T) { err := event.Validate(log) - expectedErr := "a single event must have one of (on_init: true,interval,cron) field, received:(on_init: true,cron: `0 0 * * *`, interval: `60ns`)" + expectedErr := "a single event must have one of " assert.Error(t, err) assert.Contains(t, err.Error(), expectedErr) diff --git a/config/task_validator_test.go b/config/task_validator_test.go index 285a55c..a7e576a 100644 --- a/config/task_validator_test.go +++ b/config/task_validator_test.go @@ -1,6 +1,7 @@ package config_test import ( + "runtime" "testing" "github.com/alecthomas/assert/v2" @@ -118,7 +119,11 @@ func TestTaskValidate_CredentialLog(t *testing.T) { err := task.Validate(log) assert.NoError(t, err) - assert.Contains(t, buffer.String(), "Be careful when using credentials, in local mode you can't use credentials unless running as root") + if runtime.GOOS == "windows" { + assert.Contains(t, buffer.String(), "windows os does not have capability") + } else { + assert.Contains(t, buffer.String(), "Be careful when using credentials, in local mode you can't use credentials unless running as root") + } } func TestTaskValidate_InvalidTaskWithData(t *testing.T) { From 83b96f2ad107269883dafc5ea3bc0e9c0b1b8bed Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 18:29:33 +0330 Subject: [PATCH 06/19] minor: refactor --- cmd/root.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index f8d6ad4..d3a6331 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -64,6 +64,14 @@ func panicOnErr(err error, message string) { } func initConfig() { + if runtime.GOOS == "windows" { + viper.SetDefault("shell", "C:\\WINDOWS\\system32\\cmd.exe") + viper.SetDefault("shell_args", "/c") + } else { + viper.SetDefault("shell", "/bin/sh") + viper.SetDefault("shell_args", "-c") + } + setupEnv() if cfgFile != "" { @@ -163,13 +171,6 @@ func setupEnv() { "Cannot bind log_level env variable: %s", ) - if runtime.GOOS == "windows" { - viper.SetDefault("shell", "C:\\WINDOWS\\system32\\cmd.exe") - viper.SetDefault("shell_args", "/c") - } else { - viper.SetDefault("shell", "/bin/sh") - viper.SetDefault("shell_args", "-c") - } warnOnErr( viper.BindEnv( "shell", From 2085e6e732b7ddb116f1db416b81122e7d483cf4 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 18:32:34 +0330 Subject: [PATCH 07/19] minor --- core/os_credential/windows_credential.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/os_credential/windows_credential.go b/core/os_credential/windows_credential.go index b474550..0939d75 100644 --- a/core/os_credential/windows_credential.go +++ b/core/os_credential/windows_credential.go @@ -1,6 +1,7 @@ //go:build windows // +build windows +// Package credential is ignored on windows builds package credential import ( From 8c5bdd2744b7eb40eb1f84ec165e1f2b5562b194 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 18:35:34 +0330 Subject: [PATCH 08/19] minor: more error checking --- cmd/parser/parser.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go index c6a47ff..e0ef610 100644 --- a/cmd/parser/parser.go +++ b/cmd/parser/parser.go @@ -30,8 +30,13 @@ var ( log.Panicf("failed to marshal final config: %v", err) } hashMap := make(map[string]any) - json.Unmarshal(str, &hashMap) - ans, _ := yaml.Marshal(hashMap) + if err := json.Unmarshal(str, &hashMap); err != nil { + log.Panicf("failed to unmarshal final config: %v", err) + } + ans, err := yaml.Marshal(hashMap) + if err != nil { + log.Panicf("failed to marshal final config: %v", err) + } result := string(ans) log.Printf("output:\n%s", result) if cfg.output != "" { From 04b5bac72790a3ddba6ff8b4af4a3cfef63682fb Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 18:38:21 +0330 Subject: [PATCH 09/19] fix: lint issues --- cmd/parser/cron_string.go | 11 +++++---- cmd/parser/parser.go | 2 ++ test.yaml | 49 --------------------------------------- 3 files changed, 9 insertions(+), 53 deletions(-) delete mode 100644 test.yaml diff --git a/cmd/parser/cron_string.go b/cmd/parser/cron_string.go index d1d7a6f..7c78f16 100644 --- a/cmd/parser/cron_string.go +++ b/cmd/parser/cron_string.go @@ -26,6 +26,9 @@ func NewCronFromFile(filePath string) CronString { log.Panicf("can't open cron file: %v", err) } stat, err := file.Stat() + if err != nil { + log.Panicf("can't stat cron file: %v", err) + } content := make([]byte, stat.Size()) file.Read(content) return CronString{string(content)} @@ -70,13 +73,13 @@ func (s CronString) lines() []string { return strings.Split(s.string, "\n") } -func (c *CronString) parseAsSpec( +func (s *CronString) parseAsSpec( pattern string, hasUser bool, ) []cronSpec { envTable := make(map[string]string) specs := make([]cronSpec, 0) - lines := c.sanitize().lines() + lines := s.sanitize().lines() matcher, parser := buildMapper(hasUser, pattern) for _, line := range lines { @@ -98,11 +101,11 @@ func (c *CronString) parseAsSpec( return specs } -func (c *CronString) ParseConfig( +func (s *CronString) ParseConfig( pattern string, hasUser bool, ) *config.Config { - specs := c.parseAsSpec(pattern, hasUser) + specs := s.parseAsSpec(pattern, hasUser) cfg := &config.Config{} for _, spec := range specs { addSpec(cfg, spec) diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go index e0ef610..b8baf74 100644 --- a/cmd/parser/parser.go +++ b/cmd/parser/parser.go @@ -1,3 +1,5 @@ +// Package parser manages holds the logic behind the sub command `parse` +// this package is responsible for parsing a crontab file into valid config yaml file package parser import ( diff --git a/test.yaml b/test.yaml deleted file mode 100644 index ae55495..0000000 --- a/test.yaml +++ /dev/null @@ -1,49 +0,0 @@ -jobs: - - description: Imported from cron file - events: - - cron: '* * * * *' - hooks: {} - name: 'FromCron: * * * * *' - tasks: - - command: php -v --help - env: - PATH: C:\xampp\php; - user: tng - - command: php -v --help --params - env: - PATH: C:\xampp\php; - user: tng - - description: Imported from cron file - events: - - cron: 0 0 * * * - hooks: {} - name: 'FromCron: 0 0 * * *' - tasks: - - command: php -v --help --params - env: - PATH: C:\xampp\php; - user: tng - - command: php -v --help --params - env: - PATH: C:\xampp\php; - user: tng - - description: Imported from cron file - events: - - cron: 0 0 */2 * * - hooks: {} - name: 'FromCron: 0 0 */2 * *' - tasks: - - command: php -v --help --params - env: - PATH: C:\xampp\php; - user: tng -escription: Imported from cron file - events: - - cron: 0 0 */2 * * - hooks: {} - name: 'FromCron: 0 0 */2 * *' - tasks: - - command: php -v --help --params - env: - PATH: C:\xampp\php; - user: tng From 2127c8bca2df0fab1b2c8e905d7f0f555a86d3d9 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 18:45:02 +0330 Subject: [PATCH 10/19] minor --- cmd/parser/cron_string.go | 5 ++++- meta/github.go | 11 ++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/parser/cron_string.go b/cmd/parser/cron_string.go index 7c78f16..dc0bd2b 100644 --- a/cmd/parser/cron_string.go +++ b/cmd/parser/cron_string.go @@ -30,7 +30,10 @@ func NewCronFromFile(filePath string) CronString { log.Panicf("can't stat cron file: %v", err) } content := make([]byte, stat.Size()) - file.Read(content) + _, err = file.Read(content) + if err != nil { + log.Panicf("can't open cron file: %v", err) + } return CronString{string(content)} } diff --git a/meta/github.go b/meta/github.go index e02d47c..7ce9b0b 100644 --- a/meta/github.go +++ b/meta/github.go @@ -1,9 +1,18 @@ +// Package meta contains meta data information about this program. package meta import "fmt" +func GHUserName() string { + return "FMotalleb" +} + +func GHProjectName() string { + return "crontab-go" +} + func Project() string { - return "https://github.com/FMotalleb/crontab-go" + return fmt.Sprintf("https://github.com/%s/%s", GHUserName(), GHProjectName()) } func Issues() string { From b3e8e861ae6f7707f06c6299ad06033afe5c2206 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 18:49:50 +0330 Subject: [PATCH 11/19] minor --- core/concurrency/concurrent_pool_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/concurrency/concurrent_pool_test.go b/core/concurrency/concurrent_pool_test.go index 27b81fe..cb3a60d 100644 --- a/core/concurrency/concurrent_pool_test.go +++ b/core/concurrency/concurrent_pool_test.go @@ -58,6 +58,7 @@ func TestConcurrentPool_LockUnlockGoroutine(t *testing.T) { end1 := <-chn end2 := <-chn diff := end2 - end1 - assert.True(t, diff >= 1000) + // 10% less than the actual value, possible timing issues + assert.True(t, diff >= 900) }) } From fa1eeb7b4323500c2a404a0ecca62bdc70ef0af5 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 18:56:52 +0330 Subject: [PATCH 12/19] test --- core/concurrency/concurrent_pool_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/concurrency/concurrent_pool_test.go b/core/concurrency/concurrent_pool_test.go index cb3a60d..71a4d9f 100644 --- a/core/concurrency/concurrent_pool_test.go +++ b/core/concurrency/concurrent_pool_test.go @@ -1,6 +1,7 @@ package concurrency import ( + "fmt" "testing" "time" @@ -58,7 +59,8 @@ func TestConcurrentPool_LockUnlockGoroutine(t *testing.T) { end1 := <-chn end2 := <-chn diff := end2 - end1 - // 10% less than the actual value, possible timing issues - assert.True(t, diff >= 900) + assert.NotZero(t, diff) + fmt.Println(diff) + assert.True(t, diff >= 1000) }) } From 85b972e461f00ec46bc1fd0e414ef25d13add5a6 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 18:58:38 +0330 Subject: [PATCH 13/19] minor --- core/concurrency/concurrent_pool_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/concurrency/concurrent_pool_test.go b/core/concurrency/concurrent_pool_test.go index 71a4d9f..88d1617 100644 --- a/core/concurrency/concurrent_pool_test.go +++ b/core/concurrency/concurrent_pool_test.go @@ -1,7 +1,6 @@ package concurrency import ( - "fmt" "testing" "time" @@ -60,7 +59,6 @@ func TestConcurrentPool_LockUnlockGoroutine(t *testing.T) { end2 := <-chn diff := end2 - end1 assert.NotZero(t, diff) - fmt.Println(diff) assert.True(t, diff >= 1000) }) } From c801e43abd37aa8a453e27921f2af54826e7134f Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 21:09:19 +0330 Subject: [PATCH 14/19] refactor --- cmd/parser/cron_line.go | 18 +++-- cmd/parser/cron_parser.go | 13 ---- cmd/parser/cron_spec.go | 14 ++-- cmd/parser/cron_string.go | 71 ++++++++--------- cmd/parser/helper.go | 27 +++++++ cmd/parser/parser.go | 98 +++++++++++++++--------- cmd/root.go | 5 +- core/concurrency/concurrent_pool.go | 6 +- core/concurrency/concurrent_pool_test.go | 17 ++-- core/jobs/runner.go | 7 +- main.go | 3 +- 11 files changed, 158 insertions(+), 121 deletions(-) delete mode 100644 cmd/parser/cron_parser.go create mode 100644 cmd/parser/helper.go diff --git a/cmd/parser/cron_line.go b/cmd/parser/cron_line.go index d61105d..083d8a1 100644 --- a/cmd/parser/cron_line.go +++ b/cmd/parser/cron_line.go @@ -1,7 +1,7 @@ package parser import ( - "log" + "fmt" "regexp" "strings" ) @@ -10,7 +10,9 @@ type cronLine struct { string } -func (l cronLine) exportEnv() map[string]string { +var envRegex = regexp.MustCompile(`^(?[\w\d_]+)=(?.*)$`) + +func (l cronLine) exportEnv() (map[string]string, error) { match := envRegex.FindStringSubmatch(l.string) answer := make(map[string]string) switch len(match) { @@ -18,19 +20,19 @@ func (l cronLine) exportEnv() map[string]string { case 3: answer[match[1]] = match[2] default: - log.Panicf("found multiple(%d) env vars in single line\n please attach your crontab file too\n affected line: %s\n parser result: %#v\n", len(match), l.string, match) + return nil, fmt.Errorf("unexpected line in cron file, environment regex selector cannot understand this line:\n%s", l.string) } - return answer + return answer, nil } -func (l cronLine) exportSpec(regex *regexp.Regexp, env map[string]string, parser cronSpecParser) *cronSpec { +func (l cronLine) exportSpec(regex *regexp.Regexp, env map[string]string, parser cronSpecParser) (*cronSpec, error) { match := regex.FindStringSubmatch(l.string) if len(match) < 1 { if len(strings.Trim(l.string, " \n\t")) == 0 { - return nil + return nil, nil } else { - log.Panicf("cannot parse this non-empty line as cron specification: %s", l.string) + return nil, fmt.Errorf("cannot parse this non-empty line as cron specification: %s", l.string) } } - return parser(match, env) + return parser(match, env), nil } diff --git a/cmd/parser/cron_parser.go b/cmd/parser/cron_parser.go deleted file mode 100644 index f0f1fd8..0000000 --- a/cmd/parser/cron_parser.go +++ /dev/null @@ -1,13 +0,0 @@ -package parser - -import ( - "github.com/FMotalleb/crontab-go/config" -) - -func (cfg parserConfig) parse() config.Config { - cron := NewCronFromFile(cfg.cronFile) - return *cron.ParseConfig( - cfg.cronMatcher, - cfg.hasUser, - ) -} diff --git a/cmd/parser/cron_spec.go b/cmd/parser/cron_spec.go index a8932d7..5bb0693 100644 --- a/cmd/parser/cron_spec.go +++ b/cmd/parser/cron_spec.go @@ -1,7 +1,7 @@ package parser import ( - "log" + "fmt" "regexp" ) @@ -15,12 +15,12 @@ type ( } ) -func normalParser(regex *regexp.Regexp) cronSpecParser { +func normalParser(regex *regexp.Regexp) (cronSpecParser, error) { cronIndex := regex.SubexpIndex("cron") // userIndex := regex.SubexpIndex("user") cmdIndex := regex.SubexpIndex("cmd") if cronIndex < 0 || cmdIndex < 0 { - log.Panicf("cannot find groups (cron,cmd) in regexp: `%s", regex) + return nil, fmt.Errorf("cannot find groups (cron,cmd) in regexp: `%s", regex) } return func(match []string, env map[string]string) *cronSpec { return &cronSpec{ @@ -29,15 +29,15 @@ func normalParser(regex *regexp.Regexp) cronSpecParser { command: match[cmdIndex], environ: env, } - } + }, nil } -func withUserParser(regex *regexp.Regexp) cronSpecParser { +func withUserParser(regex *regexp.Regexp) (cronSpecParser, error) { cronIndex := regex.SubexpIndex("cron") userIndex := regex.SubexpIndex("user") cmdIndex := regex.SubexpIndex("cmd") if cronIndex < 0 || cmdIndex < 0 || userIndex < 0 { - log.Panicf("cannot find groups (cron,user,cmd) in regexp: `%s", regex) + return nil, fmt.Errorf("cannot find groups (cron,user,cmd) in regexp: `%s", regex) } return func(match []string, env map[string]string) *cronSpec { return &cronSpec{ @@ -46,5 +46,5 @@ func withUserParser(regex *regexp.Regexp) cronSpecParser { command: match[cmdIndex], environ: env, } - } + }, nil } diff --git a/cmd/parser/cron_string.go b/cmd/parser/cron_string.go index dc0bd2b..ef50ba8 100644 --- a/cmd/parser/cron_string.go +++ b/cmd/parser/cron_string.go @@ -2,16 +2,14 @@ package parser import ( "fmt" - "log" - "os" "regexp" "strings" + "github.com/sirupsen/logrus" + "github.com/FMotalleb/crontab-go/config" ) -var envRegex = regexp.MustCompile(`^(?[\w\d_]+)=(?.*)$`) - type CronString struct { string } @@ -20,23 +18,6 @@ func NewCronString(cron string) CronString { return CronString{cron} } -func NewCronFromFile(filePath string) CronString { - file, err := os.OpenFile(filePath, os.O_RDONLY, 0o644) - if err != nil { - log.Panicf("can't open cron file: %v", err) - } - stat, err := file.Stat() - if err != nil { - log.Panicf("can't stat cron file: %v", err) - } - content := make([]byte, stat.Size()) - _, err = file.Read(content) - if err != nil { - log.Panicf("can't open cron file: %v", err) - } - return CronString{string(content)} -} - func (s CronString) replaceAll(regex string, repl string) CronString { reg := regexp.MustCompile(regex) out := reg.ReplaceAllString(s.string, repl) @@ -79,44 +60,57 @@ func (s CronString) lines() []string { func (s *CronString) parseAsSpec( pattern string, hasUser bool, -) []cronSpec { +) ([]cronSpec, error) { envTable := make(map[string]string) specs := make([]cronSpec, 0) lines := s.sanitize().lines() - matcher, parser := buildMapper(hasUser, pattern) - + matcher, parser, err := buildMapper(hasUser, pattern) + if err != nil { + return []cronSpec{}, err + } for _, line := range lines { l := cronLine{line} - if env := l.exportEnv(); len(env) > 0 { - for key, val := range l.exportEnv() { + if env, err := l.exportEnv(); len(env) > 0 { + if err != nil { + return nil, err + } + for key, val := range env { if old, ok := envTable[key]; ok { - log.Printf("env var of key `%s`, value `%s`, is going to be replaced by `%s`\n", key, old, val) + logrus.Warnf("env var of key `%s`, value `%s`, is going to be replaced by `%s`", key, old, val) } envTable[key] = val } } else { - if spec := l.exportSpec(matcher, envTable, parser); spec != nil { + spec, err := l.exportSpec(matcher, envTable, parser) + if err != nil { + return nil, err + } + if spec != nil { specs = append(specs, *spec) } + } } - return specs + return specs, nil } func (s *CronString) ParseConfig( pattern string, hasUser bool, -) *config.Config { - specs := s.parseAsSpec(pattern, hasUser) +) (*config.Config, error) { + specs, err := s.parseAsSpec(pattern, hasUser) + if err != nil { + return nil, err + } cfg := &config.Config{} for _, spec := range specs { addSpec(cfg, spec) } - return cfg + return cfg, nil } -func buildMapper(hasUser bool, pattern string) (*regexp.Regexp, func([]string, map[string]string) *cronSpec) { +func buildMapper(hasUser bool, pattern string) (*regexp.Regexp, cronSpecParser, error) { lineParser := "(?.*)" if hasUser { lineParser = fmt.Sprintf(`(?\w[\w\d]*)\s+%s`, lineParser) @@ -125,15 +119,18 @@ func buildMapper(hasUser bool, pattern string) (*regexp.Regexp, func([]string, m matcher, err := regexp.Compile(cronLineMatcher) if err != nil { - log.Panicf("cannot parse cron `%s`", matcher) + return nil, nil, fmt.Errorf("cannot parse cron `%s`", matcher) } var parser cronSpecParser if hasUser { - parser = withUserParser(matcher) + parser, err = withUserParser(matcher) } else { - parser = normalParser(matcher) + parser, err = normalParser(matcher) + } + if err != nil { + return nil, nil, err } - return matcher, parser + return matcher, parser, nil } func addSpec(cfg *config.Config, spec cronSpec) { diff --git a/cmd/parser/helper.go b/cmd/parser/helper.go new file mode 100644 index 0000000..8b6355b --- /dev/null +++ b/cmd/parser/helper.go @@ -0,0 +1,27 @@ +package parser + +import ( + "encoding/json" + "fmt" + + "gopkg.in/yaml.v3" + + "github.com/FMotalleb/crontab-go/config" +) + +func generateYamlFromCfg(finalConfig *config.Config) (string, error) { + str, err := json.Marshal(finalConfig) + if err != nil { + return "", fmt.Errorf("failed to marshal(json) final config: %v", err) + } + hashMap := make(map[string]any) + if err := json.Unmarshal(str, &hashMap); err != nil { + return "", fmt.Errorf("failed to unmarshal(json) final config: %v", err) + } + ans, err := yaml.Marshal(hashMap) + if err != nil { + return "", fmt.Errorf("failed to marshal(yaml) final config: %v", err) + } + result := string(ans) + return result, nil +} diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go index b8baf74..5365841 100644 --- a/cmd/parser/parser.go +++ b/cmd/parser/parser.go @@ -4,14 +4,13 @@ package parser import ( "bytes" - "encoding/json" "errors" + "fmt" "io" - "log" "os" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "gopkg.in/yaml.v3" ) var ( @@ -20,44 +19,67 @@ var ( Use: "parse ", ValidArgs: []string{"crontab file path"}, Short: "Parse crontab syntax and converts it into yaml syntax for crontab-go", - Run: func(cmd *cobra.Command, args []string) { - cfg.cronFile = cmd.Flags().Arg(0) - log.SetPrefix("[Cron Parser]") - if cfg.cronFile == "" { - log.Panicln(errors.New("no crontab file specified, see usage using --help flag")) - } - finalConfig := cfg.parse() - str, err := json.Marshal(finalConfig) - if err != nil { - log.Panicf("failed to marshal final config: %v", err) - } - hashMap := make(map[string]any) - if err := json.Unmarshal(str, &hashMap); err != nil { - log.Panicf("failed to unmarshal final config: %v", err) - } - ans, err := yaml.Marshal(hashMap) - if err != nil { - log.Panicf("failed to marshal final config: %v", err) - } - result := string(ans) - log.Printf("output:\n%s", result) - if cfg.output != "" { - outputFile, err := os.OpenFile(cfg.output, os.O_WRONLY|os.O_CREATE, 0o644) - if err != nil { - log.Panicf("failed to open output file: %v", err) - } - buf := bytes.NewBufferString(result) - _, err = io.Copy(outputFile, buf) - if err != nil { - log.Panicf("failed to write output file: %v", err) - } - } - log.Println("Done writing output") - os.Exit(0) - }, + Run: run, } ) +func run(cmd *cobra.Command, args []string) { + cfg.cronFile = cmd.Flags().Arg(0) + + cron, err := readInCron() + if err != nil { + log.Panic(err) + } + finalConfig, err := cron.ParseConfig( + cfg.cronMatcher, + cfg.hasUser, + ) + if err != nil { + log.Panicf("cannot parse given cron file: %v", err) + } + result, err := generateYamlFromCfg(finalConfig) + if err != nil { + log.Panic(err) + } + log.Printf("output:\n%s", result) + if cfg.output != "" { + outputFile, err := os.OpenFile(cfg.output, os.O_WRONLY|os.O_CREATE, 0o644) + if err != nil { + log.Panicf("failed to open output file: %v", err) + } + buf := bytes.NewBufferString(result) + _, err = io.Copy(outputFile, buf) + if err != nil { + log.Panicf("failed to write output file: %v", err) + } + } + log.Println("Done writing output") + os.Exit(0) +} + +func readInCron() (*CronString, error) { + var str string = "" + if cfg.cronFile == "" { + return nil, errors.New("please provide a cron file path, usage: `--help`") + } + file, err := os.OpenFile(cfg.cronFile, os.O_RDONLY, 0o644) + if err != nil { + return nil, fmt.Errorf("can't open cron file: %v", err) + } + stat, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("can't stat cron file: %v", err) + } + content := make([]byte, stat.Size()) + _, err = file.Read(content) + if err != nil { + return nil, fmt.Errorf("can't open cron file: %v", err) + } + str = string(content) + cron := NewCronString(str) + return &cron, nil +} + func init() { ParserCmd.PersistentFlags().StringVarP(&cfg.output, "output", "o", "", "output file to write configuration to") ParserCmd.PersistentFlags().BoolVarP(&cfg.hasUser, "with-user", "u", false, "indicates that whether the given cron file has user field") diff --git a/cmd/root.go b/cmd/root.go index d3a6331..1da58c2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -52,14 +52,13 @@ func init() { func warnOnErr(err error, message string) { if err != nil { - fmt.Printf(message, err) + logrus.Warnf(message, err) } } func panicOnErr(err error, message string) { if err != nil { - fmt.Printf(message, err) - panic(err) + logrus.Panicf(message, err) } } diff --git a/core/concurrency/concurrent_pool.go b/core/concurrency/concurrent_pool.go index 1e61449..54eb9f3 100644 --- a/core/concurrency/concurrent_pool.go +++ b/core/concurrency/concurrent_pool.go @@ -17,16 +17,16 @@ type ConcurrentPool struct { // NewConcurrentPool creates a new ConcurrentPool with the specified capacity. // It panics if the capacity is 0. -func NewConcurrentPool(capacity uint) *ConcurrentPool { +func NewConcurrentPool(capacity uint) (*ConcurrentPool, error) { if capacity == 0 { - panic(errors.New("capacity value of a concurrent poll cannot be 0")) + return nil, errors.New("capacity value of a concurrent poll cannot be 0") } return &ConcurrentPool{ internalSync: &sync.Mutex{}, available: capacity, used: 0, changeChan: make(chan interface{}), - } + }, nil } // Lock acquires a lock from the pool, waiting if necessary until a slot becomes available. diff --git a/core/concurrency/concurrent_pool_test.go b/core/concurrency/concurrent_pool_test.go index 88d1617..f3d1441 100644 --- a/core/concurrency/concurrent_pool_test.go +++ b/core/concurrency/concurrent_pool_test.go @@ -8,14 +8,14 @@ import ( ) func TestConcurrentPool_PanicCase(t *testing.T) { - assert.Panics(t, func() { - NewConcurrentPool(0) - }) + _, err := NewConcurrentPool(0) + assert.Error(t, err) } func TestConcurrentPool_LockUnlock(t *testing.T) { t.Run("Lock and Unlock with capacity 1", func(t *testing.T) { - pool := NewConcurrentPool(1) + pool, err := NewConcurrentPool(1) + assert.NoError(t, err) pool.Lock() assert.Equal(t, 1, pool.access(get)) pool.Unlock() @@ -23,7 +23,8 @@ func TestConcurrentPool_LockUnlock(t *testing.T) { }) t.Run("Lock and Unlock with capacity 2", func(t *testing.T) { - pool := NewConcurrentPool(2) + pool, err := NewConcurrentPool(2) + assert.NoError(t, err) pool.Lock() assert.Equal(t, 1, pool.access(get)) pool.Lock() @@ -35,14 +36,16 @@ func TestConcurrentPool_LockUnlock(t *testing.T) { }) t.Run("Unlock on a totally free pool", func(t *testing.T) { - pool := NewConcurrentPool(1) + pool, err := NewConcurrentPool(1) + assert.NoError(t, err) assert.Panics(t, pool.Unlock) }) } func TestConcurrentPool_LockUnlockGoroutine(t *testing.T) { t.Run("Lock and Unlock with capacity 1 (inside 2 goroutine)", func(t *testing.T) { - pool := NewConcurrentPool(1) + pool, err := NewConcurrentPool(1) + assert.NoError(t, err) chn := make(chan int64) for i := 0; i < 2; i++ { diff --git a/core/jobs/runner.go b/core/jobs/runner.go index 702d43a..b69512b 100644 --- a/core/jobs/runner.go +++ b/core/jobs/runner.go @@ -2,7 +2,6 @@ package jobs import ( "context" - "sync" "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" @@ -26,8 +25,10 @@ func InitializeJobs(log *logrus.Entry, cronInstance *cron.Cron) { c := context.Background() c = context.WithValue(c, ctxutils.JobKey, job) - var lock sync.Locker = concurrency.NewConcurrentPool(job.Concurrency) - + lock, err := concurrency.NewConcurrentPool(job.Concurrency) + if err != nil { + log.Panicf("failed to validate job (%s): %v", job.Name, err) + } logger := initLogger(c, log, job) logger = logger.WithField("concurrency", job.Concurrency) if err := job.Validate(log); err != nil { diff --git a/main.go b/main.go index bd0cb17..1a45c65 100644 --- a/main.go +++ b/main.go @@ -34,8 +34,7 @@ func main() { defer func() { if err := recover(); err != nil { log.Printf( - "recovering from a panic:\n%v\nif you think this is an error from application please report at: %s", - err, + "recovering from a panic if you think this is an error from application please report at: %s", meta.Issues(), ) os.Exit(1) From 71d6cebbb35b0fa0cbc07a2837e3a82d4629e356 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 21:10:10 +0330 Subject: [PATCH 15/19] minor: refactor --- cmd/parser/parser.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go index 5365841..4c906c2 100644 --- a/cmd/parser/parser.go +++ b/cmd/parser/parser.go @@ -25,7 +25,6 @@ var ( func run(cmd *cobra.Command, args []string) { cfg.cronFile = cmd.Flags().Arg(0) - cron, err := readInCron() if err != nil { log.Panic(err) @@ -43,20 +42,24 @@ func run(cmd *cobra.Command, args []string) { } log.Printf("output:\n%s", result) if cfg.output != "" { - outputFile, err := os.OpenFile(cfg.output, os.O_WRONLY|os.O_CREATE, 0o644) - if err != nil { - log.Panicf("failed to open output file: %v", err) - } - buf := bytes.NewBufferString(result) - _, err = io.Copy(outputFile, buf) - if err != nil { - log.Panicf("failed to write output file: %v", err) - } + writeOutput(result) } log.Println("Done writing output") os.Exit(0) } +func writeOutput(result string) { + outputFile, err := os.OpenFile(cfg.output, os.O_WRONLY|os.O_CREATE, 0o644) + if err != nil { + log.Panicf("failed to open output file: %v", err) + } + buf := bytes.NewBufferString(result) + _, err = io.Copy(outputFile, buf) + if err != nil { + log.Panicf("failed to write output file: %v", err) + } +} + func readInCron() (*CronString, error) { var str string = "" if cfg.cronFile == "" { From d1aa9724e8820dd8e1528477b47734a59a6ba445 Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 21:56:28 +0330 Subject: [PATCH 16/19] test: env line exporter tests --- cmd/parser/cron_line.go | 5 +++- cmd/parser/cron_line_test.go | 53 ++++++++++++++++++++++++++++++++++++ cmd/parser/parser.go | 8 +++--- 3 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 cmd/parser/cron_line_test.go diff --git a/cmd/parser/cron_line.go b/cmd/parser/cron_line.go index 083d8a1..e1137fe 100644 --- a/cmd/parser/cron_line.go +++ b/cmd/parser/cron_line.go @@ -20,7 +20,10 @@ func (l cronLine) exportEnv() (map[string]string, error) { case 3: answer[match[1]] = match[2] default: - return nil, fmt.Errorf("unexpected line in cron file, environment regex selector cannot understand this line:\n%s", l.string) + return nil, fmt.Errorf("unexpected response from environment parser for line:\n%s", l.string) + } + if len(answer) != 1 && len(strings.Trim(l.string, " \n\t")) != 0 { + return nil, fmt.Errorf("line cannot be parsed as environment:\n%s", l.string) } return answer, nil } diff --git a/cmd/parser/cron_line_test.go b/cmd/parser/cron_line_test.go new file mode 100644 index 0000000..5f47b84 --- /dev/null +++ b/cmd/parser/cron_line_test.go @@ -0,0 +1,53 @@ +package parser + +import ( + "testing" + + "github.com/alecthomas/assert/v2" +) + +func TestExportEnv_SingleMatch(t *testing.T) { + tests := []struct { + name string + input string + expected map[string]string + wantErr bool + }{ + { + name: "Single match", + input: "MY_VAR=value", + expected: map[string]string{"MY_VAR": "value"}, + wantErr: false, + }, + { + name: "No match", + input: "no_match", + expected: nil, + wantErr: true, + }, + { + name: "Empty input", + input: "", + expected: map[string]string{}, + wantErr: false, + }, + { + name: "Special characters", + input: "VAR_WITH_UNDERSCORE=value_with_underscore", + expected: map[string]string{"VAR_WITH_UNDERSCORE": "value_with_underscore"}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cl := cronLine{string: tt.input} + got, err := cl.exportEnv() + if tt.wantErr { + assert.Error(t, err) + } else { + assert.Equal(t, tt.expected, got) + } + }) + } +} diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go index 4c906c2..69c9278 100644 --- a/cmd/parser/parser.go +++ b/cmd/parser/parser.go @@ -25,7 +25,7 @@ var ( func run(cmd *cobra.Command, args []string) { cfg.cronFile = cmd.Flags().Arg(0) - cron, err := readInCron() + cron, err := readInCron(cfg) if err != nil { log.Panic(err) } @@ -42,13 +42,13 @@ func run(cmd *cobra.Command, args []string) { } log.Printf("output:\n%s", result) if cfg.output != "" { - writeOutput(result) + writeOutput(cfg, result) } log.Println("Done writing output") os.Exit(0) } -func writeOutput(result string) { +func writeOutput(cfg *parserConfig, result string) { outputFile, err := os.OpenFile(cfg.output, os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { log.Panicf("failed to open output file: %v", err) @@ -60,7 +60,7 @@ func writeOutput(result string) { } } -func readInCron() (*CronString, error) { +func readInCron(cfg *parserConfig) (*CronString, error) { var str string = "" if cfg.cronFile == "" { return nil, errors.New("please provide a cron file path, usage: `--help`") From 6004c79c72371cf7fafe75417405e6f56efd488a Mon Sep 17 00:00:00 2001 From: Motalleb Fallahnezhad Date: Fri, 28 Jun 2024 23:03:05 +0330 Subject: [PATCH 17/19] fix: some minor issues --- .vscode/launch.json | 3 +- cmd/parser/cron_string.go | 16 ++++++-- cmd/parser/parser.go | 5 ++- config.local.yaml | 85 ++++++++++++++++++++++++++++++++++----- config/config.go | 12 +++--- crontab.example | 41 +++++++++++++++++++ main.go | 6 ++- 7 files changed, 143 insertions(+), 25 deletions(-) create mode 100644 crontab.example diff --git a/.vscode/launch.json b/.vscode/launch.json index 1cbe3fc..840a8f5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -25,8 +25,7 @@ "env": {}, "args": [ "parse", - "crontab.log", - "-u" + "crontab.example", ] }, ] diff --git a/cmd/parser/cron_string.go b/cmd/parser/cron_string.go index ef50ba8..a707db2 100644 --- a/cmd/parser/cron_string.go +++ b/cmd/parser/cron_string.go @@ -158,10 +158,18 @@ func initJob(jobName string, timing string, cfg *config.Config) { job.Name = jobName job.Description = "Imported from cron file" job.Disabled = false - job.Events = []config.JobEvent{ - { - Cron: timing, - }, + if strings.Contains(timing, "@reboot") { + job.Events = []config.JobEvent{ + { + OnInit: true, + }, + } + } else { + job.Events = []config.JobEvent{ + { + Cron: timing, + }, + } } cfg.Jobs = append(cfg.Jobs, job) } diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go index 69c9278..08a5a67 100644 --- a/cmd/parser/parser.go +++ b/cmd/parser/parser.go @@ -26,6 +26,9 @@ var ( func run(cmd *cobra.Command, args []string) { cfg.cronFile = cmd.Flags().Arg(0) cron, err := readInCron(cfg) + log.SetFormatter(&log.TextFormatter{ + ForceColors: true, + }) if err != nil { log.Panic(err) } @@ -86,5 +89,5 @@ func readInCron(cfg *parserConfig) (*CronString, error) { func init() { ParserCmd.PersistentFlags().StringVarP(&cfg.output, "output", "o", "", "output file to write configuration to") ParserCmd.PersistentFlags().BoolVarP(&cfg.hasUser, "with-user", "u", false, "indicates that whether the given cron file has user field") - ParserCmd.PersistentFlags().StringVar(&cfg.cronMatcher, "matcher", `(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\d+(ns|us|µs|ms|s|m|h))+)|((((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|\*|(\*\/\d)) ?){5,7})`, "matcher for cron") + ParserCmd.PersistentFlags().StringVar(&cfg.cronMatcher, "matcher", `(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\d+(ns|us|µs|ms|s|m|h))+)|((((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|\*|(\*\/\d))\s*){5,7})`, "matcher for cron") } diff --git a/config.local.yaml b/config.local.yaml index 39a6bc5..104538e 100644 --- a/config.local.yaml +++ b/config.local.yaml @@ -1,18 +1,81 @@ # yaml-language-server: $schema=schema.json #TODO: unix/tcp socket controller #TODO: prometheus exporter - jobs: - - name: Test Job + - description: Imported from cron file + events: + - cron: 0 * * * * + hooks: {} + name: "FromCron: 0 * * * *" + tasks: + - command: /usr/bin/wget -O - -q -t 1 http://localhost/cron.php + - description: Imported from cron file + events: + - cron: "* * * * *" + hooks: {} + name: "FromCron: * * * * *" + tasks: + - command: /var/www/devdaily.com/bin/check-apache.sh + - description: Imported from cron file + events: + - cron: 5 10,22 * * * + hooks: {} + name: "FromCron: 5 10,22 * * *" + tasks: + - command: /var/www/devdaily.com/bin/mk-new-links.php + - description: Imported from cron file + events: + - cron: 30 4 * * * + hooks: {} + name: "FromCron: 30 4 * * *" + tasks: + - command: /var/www/devdaily.com/bin/create-all-backups.sh + - description: Imported from cron file + events: + - cron: 5 0,4,10,16 * * * + hooks: {} + name: "FromCron: 5 0,4,10,16 * * *" + tasks: + - command: /var/www/devdaily.com/bin/create-cat-list.sh + - description: Imported from cron file + events: + - cron: 5 0 * * * + hooks: {} + name: "FromCron: 5 0 * * *" + tasks: + - command: /var/www/devdaily.com/bin/resetContactForm.sh + - description: Imported from cron file + events: + - cron: 0,20,40 * * * * + hooks: {} + name: "FromCron: 0,20,40 * * * *" + tasks: + - command: /var/www/bin/ads/freshMint.sh + - description: Imported from cron file + events: + - cron: 5,25,45 * * * * + hooks: {} + name: "FromCron: 5,25,45 * * * *" tasks: - - command: echo 1 - retry-delay: 5s - # connections: - # - image: docker-mysql-database-phpmyadmin - # volumes: - # - "/home/motalleb/Downloads:/var/local/test" - # env: - # SHELL: /bin/bash + - command: /var/www/bin/ads/greenTaffy.sh + - description: Imported from cron file + events: + - cron: 10,30,50 * * * * + hooks: {} + name: "FromCron: 10,30,50 * * * *" + tasks: + - command: /var/www/bin/ads/raspberry.sh + - description: Imported from cron file + events: + - cron: 15,35,55 * * * * + hooks: {} + name: "FromCron: 15,35,55 * * * *" + tasks: + - command: /var/www/bin/ads/robinsEgg.sh + - description: Imported from cron file events: - on-init: true - - web-event: test + hooks: {} + name: "FromCron: @reboot" + tasks: + - command: test diff --git a/config/config.go b/config/config.go index 74c671e..a47ba40 100644 --- a/config/config.go +++ b/config/config.go @@ -44,8 +44,8 @@ type JobConfig struct { type JobEvent struct { Cron string `mapstructure:"cron" json:"cron,omitempty"` Interval time.Duration `mapstructure:"interval" json:"interval,omitempty"` - OnInit bool `mapstructure:"on-init" json:"on_init,omitempty"` - WebEvent string `mapstructure:"web-event" json:"web_event,omitempty"` + OnInit bool `mapstructure:"on-init" json:"on-init,omitempty"` + WebEvent string `mapstructure:"web-event" json:"web-event,omitempty"` } // JobHooks represents the hooks configuration for a job. @@ -64,7 +64,7 @@ type Task struct { // Command params Command string `mapstructure:"command" json:"command,omitempty"` - WorkingDirectory string `mapstructure:"working-dir" json:"working_directory,omitempty"` + WorkingDirectory string `mapstructure:"working-dir" json:"working-directory,omitempty"` UserName string `mapstructure:"user" json:"user,omitempty"` GroupName string `mapstructure:"group" json:"group,omitempty"` Env map[string]string `mapstructure:"env" json:"env,omitempty"` @@ -72,12 +72,12 @@ type Task struct { // Retry & Timeout config Retries uint `mapstructure:"retries" json:"retries,omitempty"` - RetryDelay time.Duration `mapstructure:"retry-delay" json:"retry_delay,omitempty"` + RetryDelay time.Duration `mapstructure:"retry-delay" json:"retry-delay,omitempty"` Timeout time.Duration `mapstructure:"timeout" json:"timeout,omitempty"` // Hooks - OnDone []Task `mapstructure:"on-done" json:"on_done,omitempty"` - OnFail []Task `mapstructure:"on-fail" json:"on_fail,omitempty"` + OnDone []Task `mapstructure:"on-done" json:"on-done,omitempty"` + OnFail []Task `mapstructure:"on-fail" json:"on-fail,omitempty"` } // TaskConnection represents the connection configuration for a task. diff --git a/crontab.example b/crontab.example new file mode 100644 index 0000000..21aa22e --- /dev/null +++ b/crontab.example @@ -0,0 +1,41 @@ +#-------------------------------------------------- +# example unix/linux crontab file format: +#-------------------------------------------------- +# min,hour,dayOfMonth,month,dayOfWeek command +# +# field allowed values +# ----- -------------- +# minute 0-59 +# hour 0-23 +# day of month 1-31 +# month 1-12 (or names, see below) +# day of week 0-7 (0 or 7 is Sun, or use names) +# +#-------------------------------------------------- + +# run the drupal cron process every hour of every day +0 * * * * /usr/bin/wget -O - -q -t 1 http://localhost/cron.php + +# run this apache kludge every minute of every day +* * * * * /var/www/devdaily.com/bin/check-apache.sh + +# generate links to new blog posts twice a day +5 10,22 * * * /var/www/devdaily.com/bin/mk-new-links.php + +# run the backup scripts at 4:30am +30 4 * * * /var/www/devdaily.com/bin/create-all-backups.sh + +# re-generate the blog "categories" list (four times a day) +5 0,4,10,16 * * * /var/www/devdaily.com/bin/create-cat-list.sh + +# reset the contact form just after midnight +5 0 * * * /var/www/devdaily.com/bin/resetContactForm.sh + +# rotate the ad banners every five minutes + +0,20,40 * * * * /var/www/bin/ads/freshMint.sh +5,25,45 * * * * /var/www/bin/ads/greenTaffy.sh +10,30,50 * * * * /var/www/bin/ads/raspberry.sh +15,35,55 * * * * /var/www/bin/ads/robinsEgg.sh + +@reboot test \ No newline at end of file diff --git a/main.go b/main.go index 1a45c65..fab7bf8 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "os" "github.com/robfig/cron/v3" + "github.com/sirupsen/logrus" "github.com/FMotalleb/crontab-go/cmd" "github.com/FMotalleb/crontab-go/core/global" @@ -31,10 +32,13 @@ import ( ) func main() { + logrus.SetFormatter(&logrus.TextFormatter{ + ForceColors: true, + }) defer func() { if err := recover(); err != nil { log.Printf( - "recovering from a panic if you think this is an error from application please report at: %s", + "an error stopped application from working, if you think this is an error in application side please report to %s", meta.Issues(), ) os.Exit(1) From 1aca175ac007161687a3990878064d2e73e3df36 Mon Sep 17 00:00:00 2001 From: FMotalleb Date: Sat, 29 Jun 2024 08:31:17 +0330 Subject: [PATCH 18/19] minor: fix lint --- cmd/parser/cron_string.go | 2 -- cmd/parser/parser.go | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/parser/cron_string.go b/cmd/parser/cron_string.go index a707db2..7f4e908 100644 --- a/cmd/parser/cron_string.go +++ b/cmd/parser/cron_string.go @@ -79,7 +79,6 @@ func (s *CronString) parseAsSpec( logrus.Warnf("env var of key `%s`, value `%s`, is going to be replaced by `%s`", key, old, val) } envTable[key] = val - } } else { spec, err := l.exportSpec(matcher, envTable, parser) @@ -89,7 +88,6 @@ func (s *CronString) parseAsSpec( if spec != nil { specs = append(specs, *spec) } - } } return specs, nil diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go index 08a5a67..7f231ac 100644 --- a/cmd/parser/parser.go +++ b/cmd/parser/parser.go @@ -64,7 +64,6 @@ func writeOutput(cfg *parserConfig, result string) { } func readInCron(cfg *parserConfig) (*CronString, error) { - var str string = "" if cfg.cronFile == "" { return nil, errors.New("please provide a cron file path, usage: `--help`") } @@ -81,7 +80,7 @@ func readInCron(cfg *parserConfig) (*CronString, error) { if err != nil { return nil, fmt.Errorf("can't open cron file: %v", err) } - str = string(content) + str := string(content) cron := NewCronString(str) return &cron, nil } From 63204c5023c680796739f7c808b8093d39a0bd46 Mon Sep 17 00:00:00 2001 From: FMotalleb Date: Sun, 30 Jun 2024 11:18:49 +0330 Subject: [PATCH 19/19] fix: set logger output to stderr and return config to stdout --- .vscode/launch.json | 1 + .vscode/settings.json | 1 + cmd/parser/cron_string.go | 37 ++++++++++++++++++++++++++----------- cmd/parser/parser.go | 14 +++++++++++--- 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 840a8f5..0719769 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -26,6 +26,7 @@ "args": [ "parse", "crontab.example", + "--verbose", ] }, ] diff --git a/.vscode/settings.json b/.vscode/settings.json index f313517..034569e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -58,6 +58,7 @@ "staticcheck", "stylecheck", "Tracef", + "Traceln", "typecheck", "unconvert", "unparam", diff --git a/cmd/parser/cron_string.go b/cmd/parser/cron_string.go index 7f4e908..d756451 100644 --- a/cmd/parser/cron_string.go +++ b/cmd/parser/cron_string.go @@ -5,7 +5,7 @@ import ( "regexp" "strings" - "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" "github.com/FMotalleb/crontab-go/config" ) @@ -46,11 +46,20 @@ func (s CronString) sanitizeComments() CronString { } func (s CronString) sanitize() CronString { - return s. + sane := s. replaceAll("\r\n", "\n"). sanitizeComments(). sanitizeLineBreaker(). sanitizeEmptyLine() + log.TraceFn(func() []interface{} { + return []any{ + "sanitizing input:\n", + s.string, + "\nOutput:\n", + sane.string, + } + }) + return sane } func (s CronString) lines() []string { @@ -65,18 +74,20 @@ func (s *CronString) parseAsSpec( specs := make([]cronSpec, 0) lines := s.sanitize().lines() matcher, parser, err := buildMapper(hasUser, pattern) + log.Tracef("parsing lines using `%s` line matcher", matcher.String()) if err != nil { return []cronSpec{}, err } - for _, line := range lines { + for num, line := range lines { l := cronLine{line} if env, err := l.exportEnv(); len(env) > 0 { + log.Tracef("line %d(post sanitize) is identified as environment line", num) if err != nil { return nil, err } for key, val := range env { if old, ok := envTable[key]; ok { - logrus.Warnf("env var of key `%s`, value `%s`, is going to be replaced by `%s`", key, old, val) + log.Warnf("env var of key `%s`, value `%s`, is going to be replaced by `%s`", key, old, val) } envTable[key] = val } @@ -113,24 +124,28 @@ func buildMapper(hasUser bool, pattern string) (*regexp.Regexp, cronSpecParser, if hasUser { lineParser = fmt.Sprintf(`(?\w[\w\d]*)\s+%s`, lineParser) } + cronLineMatcher := fmt.Sprintf(`^(?%s)\s+%s$`, pattern, lineParser) matcher, err := regexp.Compile(cronLineMatcher) if err != nil { - return nil, nil, fmt.Errorf("cannot parse cron `%s`", matcher) - } - var parser cronSpecParser - if hasUser { - parser, err = withUserParser(matcher) - } else { - parser, err = normalParser(matcher) + return nil, nil, fmt.Errorf("failed to compile cron line parser regexp: `%s`", matcher) } + parser, err := getLineParser(hasUser, matcher) if err != nil { return nil, nil, err } return matcher, parser, nil } +func getLineParser(hasUser bool, matcher *regexp.Regexp) (cronSpecParser, error) { + if hasUser { + return withUserParser(matcher) + } else { + return normalParser(matcher) + } +} + func addSpec(cfg *config.Config, spec cronSpec) { jobName := fmt.Sprintf("FromCron: %s", spec.timing) for _, job := range cfg.Jobs { diff --git a/cmd/parser/parser.go b/cmd/parser/parser.go index 7f231ac..8e791f0 100644 --- a/cmd/parser/parser.go +++ b/cmd/parser/parser.go @@ -24,11 +24,17 @@ var ( ) func run(cmd *cobra.Command, args []string) { - cfg.cronFile = cmd.Flags().Arg(0) - cron, err := readInCron(cfg) log.SetFormatter(&log.TextFormatter{ ForceColors: true, }) + log.SetOutput(os.Stderr) + cfg.cronFile = cmd.Flags().Arg(0) + + if trace, err := cmd.Flags().GetBool("verbose"); err == nil && trace { + log.SetLevel(log.TraceLevel) + } + log.Traceln("source file: ", cfg.cronFile) + cron, err := readInCron(cfg) if err != nil { log.Panic(err) } @@ -43,7 +49,8 @@ func run(cmd *cobra.Command, args []string) { if err != nil { log.Panic(err) } - log.Printf("output:\n%s", result) + fmt.Println("# yaml-language-server: $schema=https://raw.githubusercontent.com/FMotalleb/crontab-go/main/schema.json") + fmt.Println(result) if cfg.output != "" { writeOutput(cfg, result) } @@ -88,5 +95,6 @@ func readInCron(cfg *parserConfig) (*CronString, error) { func init() { ParserCmd.PersistentFlags().StringVarP(&cfg.output, "output", "o", "", "output file to write configuration to") ParserCmd.PersistentFlags().BoolVarP(&cfg.hasUser, "with-user", "u", false, "indicates that whether the given cron file has user field") + ParserCmd.PersistentFlags().BoolP("verbose", "v", false, "sets the logging level to trace and verbose logging") ParserCmd.PersistentFlags().StringVar(&cfg.cronMatcher, "matcher", `(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\d+(ns|us|µs|ms|s|m|h))+)|((((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|\*|(\*\/\d))\s*){5,7})`, "matcher for cron") }