diff --git a/.env.example b/.env.example index 2d6917a..720f00a 100644 --- a/.env.example +++ b/.env.example @@ -4,9 +4,6 @@ LOG_FILE=/var/log/crontab-go.log LOG_STDOUT=true LOG_LEVEL=debug -SHELL=/bin/bash -SHELL_ARGS=-c - WEBSERVER_ADDRESS= WEBSERVER_PORT= WEBSERVER_USERNAME=admin @@ -14,3 +11,12 @@ WEBSERVER_PASSWORD=f2f9899c-567c-455f-8a82-77a2c66e736e WEBSERVER_METRICS=true TZ=Asia/Tehran + +# defaults to sh on linux and cmd on windows +SHELL=/bin/bash +# can be an array (`:` seperated) the seperation can be escaped using `\:` +SHELL_ARGS=-c + +# env value will enable the `CRONTAB_GO_EVENT_ARGUMENTS` environments variable and event data will be passed using this env +# Can be [none (default), env, arg] any other value will not break the program but will be oprational as default value +SHELL_ARG_COMPATIBILITY=none diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 491e9c6..a42ff5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,21 +28,24 @@ jobs: run: make ci - name: Upload coverage uses: actions/upload-artifact@v4 + if: ${{ github.event_name != 'pull_request' }} with: name: coverage-${{ matrix.os }} path: coverage.* - run: goreleaser release --rm-dist --snapshot - if: ${{ runner.os == 'Linux' }} + if: ${{ runner.os == 'Linux' && github.event_name != 'pull_request' }} - name: Upload dist uses: actions/upload-artifact@v4 + if: ${{ github.event_name != 'pull_request' }} with: name: dist-${{ matrix.os }} path: dist - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 + if: ${{ github.event_name != 'pull_request' }} with: fail_ci_if_error: true file: ./coverage.out diff --git a/.golangci.yml b/.golangci.yml index d67e1b5..e7a3773 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,6 +1,6 @@ linters-settings: funlen: - lines: 100 + lines: 150 statements: 50 gocyclo: min-complexity: 15 diff --git a/cmd/root.go b/cmd/root.go index e199c87..9cf9ca7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,7 +40,7 @@ func Execute() { } func init() { - warnOnErr(godotenv.Load(), "Cannot initialize .env file: %s") + _ = godotenv.Load() rootCmd.AddCommand(parser.ParserCmd) rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is config.yaml)") @@ -188,6 +188,12 @@ func setupEnv() { ), "Cannot bind shell_args env variable: %s", ) + warnOnErr( + viper.BindEnv( + "shell_arg_compatibility", + ), + "Cannot bind shell_arg_compatibility env variable: %s", + ) viper.AutomaticEnv() } diff --git a/config.local.yaml b/config.local.yaml index 6b6da74..dc00854 100644 --- a/config.local.yaml +++ b/config.local.yaml @@ -3,7 +3,7 @@ jobs: - name: echo tasks: - - command: echo "$0 $@" + - command: echo "$CRONTAB_GO_EVENT_ARGUMENTS" env: "COLE": test events: diff --git a/config/config.go b/config/config.go index 445cded..a16413a 100644 --- a/config/config.go +++ b/config/config.go @@ -28,6 +28,8 @@ type Config struct { WebServerPassword string `mapstructure:"webserver_password" json:"webserver_password,omitempty"` WebServerMetrics bool `mapstructure:"webserver_metrics" json:"webserver_metrics,omitempty"` + ShellArgCompatibility ShellArgCompatibilityMode `mapstructure:"shell_arg_compatibility" json:"shell_arg_compatibility,omitempty"` + Jobs []*JobConfig `mapstructure:"jobs" json:"jobs"` } @@ -109,3 +111,12 @@ type TaskConnection struct { Volumes []string `mapstructure:"volumes" json:"volumes,omitempty"` Networks []string `mapstructure:"networks" json:"networks,omitempty"` } + +type ShellArgCompatibilityMode string + +const ( + DefaultShellArgCompatibility ShellArgCompatibilityMode = EventArgOmit + EventArgOmit ShellArgCompatibilityMode = "none" + EventArgPassingAsArgs ShellArgCompatibilityMode = "arg" + EventArgPassingAsEnviron ShellArgCompatibilityMode = "env" +) diff --git a/core/cmd_connection/cmd_utils/cmd_context.go b/core/cmd_connection/cmd_utils/cmd_context.go new file mode 100644 index 0000000..a3226c2 --- /dev/null +++ b/core/cmd_connection/cmd_utils/cmd_context.go @@ -0,0 +1,144 @@ +// Package cmdutils contain helper methods for cmd executors +package cmdutils + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/sirupsen/logrus" + + "github.com/FMotalleb/crontab-go/config" + "github.com/FMotalleb/crontab-go/core/utils" + "github.com/FMotalleb/crontab-go/ctxutils" +) + +type Ctx struct { + context.Context + logger *logrus.Entry +} + +func NewCtx( + ctx context.Context, + taskEnviron map[string]string, + logger *logrus.Entry, +) *Ctx { + result := &Ctx{ + Context: ctx, + logger: logger, + } + result.init(taskEnviron) + return result +} + +func (ctx *Ctx) init(taskEnviron map[string]string) { + osEnviron := os.Environ() + ctx.logger.Trace("Initial environment variables: ", osEnviron) + ctx.Context = context.WithValue( + ctx, + ctxutils.Environments, + map[string]string{}, + ) + for _, pair := range osEnviron { + parts := strings.SplitN(pair, "=", 2) + if len(parts) == 2 { + ctx.envAdd(parts[0], parts[1]) + } + } + for key, val := range taskEnviron { + ctx.envAdd(key, val) + switch strings.ToLower(key) { + case "shell": + ctx.logger.Info("you've used `SHELL` env variable in command environments, overriding the global shell with:", val) + case "shell_args": + ctx.logger.Info("you've used `SHELL_ARGS` env variable in command environments, overriding the global shell_args with: ", val) + case "shell_arg_compatibility": + ctx.logger.Info("you've used `SHELL_ARG_COMPATIBILITY` env variable in command environments, overriding the global shell_arg_compatibility with: ", val) + } + } +} + +func (ctx *Ctx) envGetAll() map[string]string { + if env := ctx.Value(ctxutils.Environments); env != nil { + return env.(map[string]string) + } + return map[string]string{} +} + +func (ctx *Ctx) envGet(key string) string { + return ctx.envGetAll()[key] +} + +func (ctx *Ctx) envAdd(key string, value string) { + oldEnv := ctx.envGetAll() + key = strings.ToUpper(key) + oldEnv[key] = value + ctx.Context = context.WithValue( + ctx, + ctxutils.Environments, + oldEnv, + ) +} + +func (ctx *Ctx) envReshape() []string { + env := ctx.envGetAll() + var result []string + for key, val := range env { + result = append(result, fmt.Sprintf("%s=%s", strings.ToUpper(key), val)) + } + return result +} + +func (ctx *Ctx) getShell() string { + return ctx.envGet("SHELL") +} + +func (ctx *Ctx) getShellArg() string { + return ctx.envGet("SHELL_ARGS") +} + +func (ctx *Ctx) getShellArgCompatibility() config.ShellArgCompatibilityMode { + result := config.ShellArgCompatibilityMode(ctx.envGet("SHELL_ARG_COMPATIBILITY")) + switch result { + case "": + return config.DefaultShellArgCompatibility + default: + return result + } +} + +func (ctx *Ctx) BuildExecuteParams(command string, eventData []string) (shell string, cmd []string, env []string) { + environments := ctx.envReshape() + shell = ctx.getShell() + shellArgs := utils.EscapedSplit(ctx.getShellArg(), ':') + shellArgs = append(shellArgs, command) + switch ctx.getShellArgCompatibility() { + case config.EventArgOmit: + ctx.logger.Debug("event arguments will not be passed to the command") + case config.EventArgPassingAsArgs: + shellArgs = append(shellArgs, eventData...) + case config.EventArgPassingAsEnviron: + environments = append( + environments, + fmt.Sprintf("CRONTAB_GO_EVENT_ARGUMENTS=%s", + collectEventForEnv(eventData), + ), + ) + default: + ctx.logger.Warn("event argument passing mode is not supported, using default mode (omitting)") + } + return shell, shellArgs, environments +} + +func collectEventForEnv(eventData []string) string { + builder := &strings.Builder{} + for i, part := range eventData { + builder.WriteString(strings.ReplaceAll(part, ":", "\\:")) + if i < len(eventData)-1 { + builder.WriteRune(':') + } + } + + return builder.String() +} diff --git a/core/cmd_connection/docker_attach.go b/core/cmd_connection/docker_attach.go index cf7e2af..1e61819 100644 --- a/core/cmd_connection/docker_attach.go +++ b/core/cmd_connection/docker_attach.go @@ -11,6 +11,7 @@ import ( "github.com/FMotalleb/crontab-go/abstraction" "github.com/FMotalleb/crontab-go/config" + cmdutils "github.com/FMotalleb/crontab-go/core/cmd_connection/cmd_utils" "github.com/FMotalleb/crontab-go/ctxutils" ) @@ -50,7 +51,7 @@ func NewDockerAttachConnection(log *logrus.Entry, conn *config.TaskConnection) a // Returns: // - An error if the preparation fails, otherwise nil. func (d *DockerAttachConnection) Prepare(ctx context.Context, task *config.Task) error { - shell, shellArgs, env := reshapeEnviron(task.Env, d.log) + cmdCtx := cmdutils.NewCtx(ctx, task.Env, d.log) d.ctx = ctx // Specify the container ID or name d.containerID = d.conn.ContainerName @@ -59,22 +60,17 @@ func (d *DockerAttachConnection) Prepare(ctx context.Context, task *config.Task) d.conn.DockerConnection = "unix:///var/run/docker.sock" } params := ctx.Value(ctxutils.EventData).([]string) + shell, shellArgs, environments := cmdCtx.BuildExecuteParams(task.Command, params) cmd := append( []string{shell}, - append( - shellArgs, - append( - []string{task.Command}, - params..., - )..., - )..., + shellArgs..., ) // Create an exec configuration d.execCFG = &container.ExecOptions{ AttachStdout: true, AttachStderr: true, Privileged: true, - Env: env, + Env: environments, WorkingDir: task.WorkingDirectory, User: task.UserName, Cmd: cmd, diff --git a/core/cmd_connection/docker_create.go b/core/cmd_connection/docker_create.go index a7a49ad..6a25a42 100644 --- a/core/cmd_connection/docker_create.go +++ b/core/cmd_connection/docker_create.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "io" - "strings" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" @@ -13,6 +12,8 @@ import ( "github.com/FMotalleb/crontab-go/abstraction" "github.com/FMotalleb/crontab-go/config" + cmdutils "github.com/FMotalleb/crontab-go/core/cmd_connection/cmd_utils" + "github.com/FMotalleb/crontab-go/core/utils" "github.com/FMotalleb/crontab-go/ctxutils" "github.com/FMotalleb/crontab-go/helpers" ) @@ -53,7 +54,7 @@ func NewDockerCreateConnection(log *logrus.Entry, conn *config.TaskConnection) a // Returns: // - An error if the preparation fails, otherwise nil. func (d *DockerCreateConnection) Prepare(ctx context.Context, task *config.Task) error { - shell, shellArgs, env := reshapeEnviron(task.Env, d.log) + cmdCtx := cmdutils.NewCtx(ctx, task.Env, d.log) d.ctx = ctx if d.conn.DockerConnection == "" { d.log.Debug("No explicit docker connection specified, using default: `unix:///var/run/docker.sock`") @@ -61,26 +62,21 @@ func (d *DockerCreateConnection) Prepare(ctx context.Context, task *config.Task) } params := ctx.Value(ctxutils.EventData).([]string) + shell, shellArgs, environments := cmdCtx.BuildExecuteParams(task.Command, params) cmd := append( []string{shell}, - append( - shellArgs, - append( - []string{task.Command}, - params..., - )..., - )..., + shellArgs..., ) volumes := make(map[string]struct{}) for _, volume := range d.conn.Volumes { - inContainer := strings.Split(volume, ":")[1] + inContainer := utils.EscapedSplit(volume, ':')[1] volumes[inContainer] = struct{}{} } // Create an exec configuration d.containerConfig = &container.Config{ AttachStdout: true, AttachStderr: true, - Env: env, + Env: environments, WorkingDir: task.WorkingDirectory, User: task.UserName, Cmd: cmd, diff --git a/core/cmd_connection/helpers.go b/core/cmd_connection/helpers.go index 48c97fb..466d65f 100644 --- a/core/cmd_connection/helpers.go +++ b/core/cmd_connection/helpers.go @@ -1,15 +1,5 @@ package connection -import ( - "fmt" - "os" - "strings" - - "github.com/sirupsen/logrus" - - "github.com/FMotalleb/crontab-go/cmd" -) - // reshapeEnviron modifies the environment variables for a given task. // It allows overriding the global shell and shell arguments with task-specific values. // @@ -21,24 +11,28 @@ import ( // - string: The shell to be used for the task, either the global shell or the overridden shell. // - []string: The shell arguments to be used for the task, either the global shell arguments or the overridden shell arguments. // - []string: The complete set of environment variables for the task, including any task-specific overrides. -func reshapeEnviron(taskEnvironments map[string]string, log *logrus.Entry) (string, []string, []string) { - shell := cmd.CFG.Shell - shellArgs := strings.Split(cmd.CFG.ShellArgs[0], ":") - env := os.Environ() - log.Trace("Initial environment variables: ", env) - for key, val := range taskEnvironments { - env = append(env, fmt.Sprintf("%s=%s", strings.ToUpper(key), val)) - oldValue := os.Getenv(key) - log.Debugf("Adding environment variable: %s=%s, before this change was: `%s`", key, val, oldValue) - switch strings.ToLower(key) { - case "shell": - log.Info("you've used `SHELL` env variable in command environments, overriding the global shell with:", val) - shell = val - case "shell_args": - log.Info("you've used `SHELL_ARGS` env variable in command environments, overriding the global shell_args with: ", val) - shellArgs = strings.Split(val, ":") - } - } - log.Trace("Final environment variables: ", env) - return shell, shellArgs, env -} +// func reshapeEnviron(ctx CmdCtx, taskEnvironments map[string]string, log *logrus.Entry) (string, []string, []string) { +// shell := cmd.CFG.Shell +// shellArgs := strings.Split(cmd.CFG.ShellArgs[0], ":") +// env := os.Environ() +// log.Trace("Initial environment variables: ", env) +// for key, val := range taskEnvironments { +// env = append(env, fmt.Sprintf("%s=%s", strings.ToUpper(key), val)) +// oldValue := os.Getenv(key) +// log.Tracef("Adding environment variable: %s=%s, before this change was: `%s`", key, val, oldValue) +// switch strings.ToLower(key) { +// case "shell": +// log.Info("you've used `SHELL` env variable in command environments, overriding the global shell with:", val) +// shell = val +// case "shell_args": +// log.Info("you've used `SHELL_ARGS` env variable in command environments, overriding the global shell_args with: ", val) +// shellArgs = strings.Split(val, ":") +// case "shell_arg_compatibility": +// log.Info("you've used `SHELL_ARG_COMPATIBILITY` env variable in command environments, overriding the global shell_arg_compatibility with: ", val) +// env = append(env, fmt.Sprintf("%s=%s", "CRONTAB_GO_EVENT_ARGUMENTS", val)) +// shellArgCompatibility = strings.Split(val, ":") +// } +// } +// log.Trace("Final environment variables: ", env) +// return shell, shellArgs, env +// } diff --git a/core/cmd_connection/local.go b/core/cmd_connection/local.go index a548d27..8807a09 100644 --- a/core/cmd_connection/local.go +++ b/core/cmd_connection/local.go @@ -12,6 +12,7 @@ import ( "github.com/FMotalleb/crontab-go/abstraction" "github.com/FMotalleb/crontab-go/config" + cmdutils "github.com/FMotalleb/crontab-go/core/cmd_connection/cmd_utils" credential "github.com/FMotalleb/crontab-go/core/os_credential" "github.com/FMotalleb/crontab-go/ctxutils" ) @@ -35,7 +36,7 @@ func NewLocalCMDConn(log *logrus.Entry) abstraction.CmdConnection { // It sets up the command with the provided context, task, and environment. // It returns an error if the preparation fails. func (l *Local) Prepare(ctx context.Context, task *config.Task) error { - shell, shellArgs, env := reshapeEnviron(task.Env, l.log) + cmdCtx := cmdutils.NewCtx(ctx, task.Env, l.log) workingDir := task.WorkingDirectory if workingDir == "" { var e error @@ -44,34 +45,30 @@ func (l *Local) Prepare(ctx context.Context, task *config.Task) error { return fmt.Errorf("cannot get current working directory: %s", e) } } - params := ctx.Value(ctxutils.EventData).([]string) + + event := ctx.Value(ctxutils.EventData).([]string) + shell, commandArg, environ := cmdCtx.BuildExecuteParams(task.Command, event) l.cmd = exec.CommandContext( ctx, shell, - append( - shellArgs, - append( - []string{task.Command}, - params..., - )..., - )..., + commandArg..., ) l.log = l.log.WithFields( logrus.Fields{ "working_directory": workingDir, "shell": shell, - "shell_args": shellArgs, + "shell_args": commandArg, }, ) credential.SetUser(l.log, l.cmd, task.UserName, task.GroupName) - l.cmd.Env = env + l.cmd.Env = environ l.cmd.Dir = workingDir // Add additional logging fields if needed l.log.WithFields(logrus.Fields{ "working_directory": workingDir, "shell": shell, - "shell_args": shellArgs, + "shell_args": commandArg, "task": task, }).Debug("command prepared") diff --git a/core/os_credential/windows_credential.go b/core/os_credential/windows_credential.go index 0939d75..fd086c6 100644 --- a/core/os_credential/windows_credential.go +++ b/core/os_credential/windows_credential.go @@ -10,11 +10,15 @@ import ( "github.com/sirupsen/logrus" ) +// Validate NOOP if user and group are empty log a warning if not and always returns nil func Validate(log *logrus.Entry, usr string, grp string) error { + if usr == "" && grp == "" { + return nil // skip warn message if no user and group provided + } log.Warn("windows os does not have capability to set user thus validation will pass but will not work") return nil } +// SetUser NOOP func SetUser(log *logrus.Entry, _ *exec.Cmd, _ string, _ string) { - log.Warn("cannot set user in windows platform") } diff --git a/core/utils/strings.go b/core/utils/strings.go new file mode 100644 index 0000000..f2d7b91 --- /dev/null +++ b/core/utils/strings.go @@ -0,0 +1,40 @@ +package utils + +const ( + escapedCharacter = '\\' +) + +func EscapedSplit(s string, sep rune) []string { + result := make([]string, 0) + buffer := make([]byte, 0) + pushBuff := func(r rune) { + buffer = append(buffer, byte(r)) + } + escaped := false + + for _, part := range s { + switch { + case escaped && part == sep: + pushBuff(part) + escaped = false + case escaped && part != sep: + pushBuff(escapedCharacter) + pushBuff(part) + escaped = false + case part == escapedCharacter: + escaped = true + case part == sep: + result = append(result, string(buffer)) + buffer = make([]byte, 0) + default: + pushBuff(part) + } + } + if len(buffer) > 0 { + result = append(result, string(buffer)) + } + if escaped { + panic("escaped character at the end of string") + } + return result +} diff --git a/core/utils/utils_test.go b/core/utils/utils_test.go index b87208f..9fe7bf0 100644 --- a/core/utils/utils_test.go +++ b/core/utils/utils_test.go @@ -162,3 +162,45 @@ func TestList(t *testing.T) { assert.True(t, list.IsNotEmpty()) }) } + +func TestEscapedSplit(t *testing.T) { + t.Run("Panic when last rune is escape rune '\\'", + func(t *testing.T) { + str := "must-panic\\" + assert.Panics(t, func() { + utils.EscapedSplit(str, '-') + }) + }, + ) + t.Run("Normal input returns slice with single item (whole input)", + func(t *testing.T) { + str := "nothing to split" + result := utils.EscapedSplit(str, '-') + assert.Equal(t, []string{str}, result) + }, + ) + t.Run("Normal input (with escaped splitter) returns slice with single item (whole input)", + func(t *testing.T) { + str := "does\\ not\\ split" + expectedResult := []string{"does not split"} + result := utils.EscapedSplit(str, ' ') + assert.Equal(t, expectedResult, result) + }, + ) + t.Run("Normal escape does nothing to non split character", + func(t *testing.T) { + str := "does\\nnot\\nsplit" + expectedResult := []string{"does\\nnot\\nsplit"} + result := utils.EscapedSplit(str, ' ') + assert.Equal(t, expectedResult, result) + }, + ) + t.Run("Normal input with and without escaped splitter returns correct slice", + func(t *testing.T) { + str := "this splits but this\\ will\\ not" + expectedResult := []string{"this", "splits", "but", "this will not"} + result := utils.EscapedSplit(str, ' ') + assert.Equal(t, expectedResult, result) + }, + ) +} diff --git a/ctxutils/keys.go b/ctxutils/keys.go index 8e8ad00..c0ffca6 100644 --- a/ctxutils/keys.go +++ b/ctxutils/keys.go @@ -12,4 +12,5 @@ var ( FailedRemotes ContextKey = ContextKey("failed-remotes") EventListeners ContextKey = ContextKey("event-listeners") EventData ContextKey = ContextKey("event-data") + Environments ContextKey = ContextKey("cmd-environments") )