diff --git a/CHANGELOG.md b/CHANGELOG.md index 1348522b..18e41b74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `orDefault()` function. [See docs](https://daseldocs.tomwright.me/functions/ordefault) +- `--csv-comma` flag to change the csv separator. +- `--csv-write-comma` flag to change the csv separator specifically for writes. +- `--csv-comment` flag to change the csv comment character. +- `--csv-crlf` flag to enable or disable CRLF output when working with csv files. ### Fixed diff --git a/internal/command/delete.go b/internal/command/delete.go index e65c070b..6ed10435 100644 --- a/internal/command/delete.go +++ b/internal/command/delete.go @@ -27,6 +27,10 @@ func deleteFlags(cmd *cobra.Command) { cmd.Flags().Bool("pretty", true, "Pretty print the output.") cmd.Flags().Bool("colour", false, "Print colourised output.") cmd.Flags().Bool("escape-html", false, "Escape HTML tags when writing output.") + cmd.Flags().String("csv-comma", ",", "Comma separator to use when working with csv files.") + cmd.Flags().String("csv-write-comma", "", "Comma separator used when writing csv files. Overrides csv-comma when writing.") + cmd.Flags().String("csv-comment", "", "Comma separator used when reading csv files.") + cmd.Flags().Bool("csv-crlf", false, "True to use CRLF when writing CSV files.") _ = cmd.MarkFlagFilename("file") } @@ -40,12 +44,18 @@ func deleteRunE(cmd *cobra.Command, args []string) error { colourFlag, _ := cmd.Flags().GetBool("colour") escapeHTMLFlag, _ := cmd.Flags().GetBool("escape-html") outFlag, _ := cmd.Flags().GetString("out") + csvComma, _ := cmd.Flags().GetString("csv-comma") + csvWriteComma, _ := cmd.Flags().GetString("csv-write-comma") + csvComment, _ := cmd.Flags().GetString("csv-comment") + csvCRLF, _ := cmd.Flags().GetBool("csv-crlf") opts := &deleteOptions{ Read: &readOptions{ - Reader: nil, - Parser: readParserFlag, - FilePath: fileFlag, + Reader: nil, + Parser: readParserFlag, + FilePath: fileFlag, + CsvComma: csvComma, + CsvComment: csvComment, }, Write: &writeOptions{ Writer: nil, @@ -54,6 +64,8 @@ func deleteRunE(cmd *cobra.Command, args []string) error { PrettyPrint: prettyPrintFlag, Colourise: colourFlag, EscapeHTML: escapeHTMLFlag, + CsvComma: csvWriteComma, + CsvUseCRLF: csvCRLF, }, Selector: selectorFlag, } diff --git a/internal/command/options.go b/internal/command/options.go index 2e980f68..82513e27 100644 --- a/internal/command/options.go +++ b/internal/command/options.go @@ -16,6 +16,10 @@ type readOptions struct { Parser string // FilePath is the path to the source file. FilePath string + // CsvComma is the comma character used when reading CSV files. + CsvComma string + // CsvComment is the comment character used when reading CSV files. + CsvComment string } func (o *readOptions) readFromStdin() bool { @@ -49,6 +53,14 @@ func (o *readOptions) rootValue(cmd *cobra.Command) (dasel.Value, error) { return dasel.Value{}, fmt.Errorf("could not get read parser: %w", err) } + options := make([]storage.ReadWriteOption, 0) + if o.CsvComma != "" { + options = append(options, storage.CsvCommaOption([]rune(o.CsvComma)[0])) + } + if o.CsvComment != "" { + options = append(options, storage.CsvCommentOption([]rune(o.CsvComment)[0])) + } + reader := o.Reader if reader == nil { if o.readFromStdin() { @@ -63,7 +75,7 @@ func (o *readOptions) rootValue(cmd *cobra.Command) (dasel.Value, error) { } } - return storage.Load(parser, reader) + return storage.Load(parser, reader, options...) } type writeOptions struct { @@ -77,6 +89,11 @@ type writeOptions struct { PrettyPrint bool Colourise bool EscapeHTML bool + + // CsvComma is the comma character used when writing CSV files. + CsvComma string + // CsvUseCRLF determines whether CRLF is used when writing CSV files. + CsvUseCRLF bool } func (o *writeOptions) writeToStdout() bool { @@ -121,6 +138,15 @@ func (o *writeOptions) writeValues(cmd *cobra.Command, readOptions *readOptions, storage.ColouriseOption(o.Colourise), storage.EscapeHTMLOption(o.EscapeHTML), storage.PrettyPrintOption(o.PrettyPrint), + storage.CsvUseCRLFOption(o.CsvUseCRLF), + } + + if o.CsvComma == "" && readOptions.CsvComma != "" { + o.CsvComma = readOptions.CsvComma + } + + if o.CsvComma != "" { + options = append(options, storage.CsvCommaOption([]rune(o.CsvComma)[0])) } writer := o.Writer diff --git a/internal/command/put.go b/internal/command/put.go index 79506672..05e5e092 100644 --- a/internal/command/put.go +++ b/internal/command/put.go @@ -32,6 +32,10 @@ func putFlags(cmd *cobra.Command) { cmd.Flags().Bool("pretty", true, "Pretty print the output.") cmd.Flags().Bool("colour", false, "Print colourised output.") cmd.Flags().Bool("escape-html", false, "Escape HTML tags when writing output.") + cmd.Flags().String("csv-comma", ",", "Comma separator to use when working with csv files.") + cmd.Flags().String("csv-write-comma", "", "Comma separator used when writing csv files. Overrides csv-comma when writing.") + cmd.Flags().String("csv-comment", "", "Comma separator used when reading csv files.") + cmd.Flags().Bool("csv-crlf", false, "True to use CRLF when writing CSV files.") _ = cmd.MarkFlagFilename("file") } @@ -47,12 +51,18 @@ func putRunE(cmd *cobra.Command, args []string) error { colourFlag, _ := cmd.Flags().GetBool("colour") escapeHTMLFlag, _ := cmd.Flags().GetBool("escape-html") outFlag, _ := cmd.Flags().GetString("out") + csvComma, _ := cmd.Flags().GetString("csv-comma") + csvWriteComma, _ := cmd.Flags().GetString("csv-write-comma") + csvComment, _ := cmd.Flags().GetString("csv-comment") + csvCRLF, _ := cmd.Flags().GetBool("csv-crlf") opts := &putOptions{ Read: &readOptions{ - Reader: nil, - Parser: readParserFlag, - FilePath: fileFlag, + Reader: nil, + Parser: readParserFlag, + FilePath: fileFlag, + CsvComma: csvComma, + CsvComment: csvComment, }, Write: &writeOptions{ Writer: nil, @@ -61,6 +71,8 @@ func putRunE(cmd *cobra.Command, args []string) error { PrettyPrint: prettyPrintFlag, Colourise: colourFlag, EscapeHTML: escapeHTMLFlag, + CsvComma: csvWriteComma, + CsvUseCRLF: csvCRLF, }, Selector: selectorFlag, ValueType: typeFlag, diff --git a/internal/command/put_test.go b/internal/command/put_test.go index aa60ff23..c2fb3856 100644 --- a/internal/command/put_test.go +++ b/internal/command/put_test.go @@ -188,4 +188,16 @@ func TestPutCommand(t *testing.T) { nil, nil, )) + + t.Run("CsvChangeSeparator", runTest( + []string{"put", "-r", "csv", "-t", "int", "-v", "5", "--csv-write-comma", ".", "[0].a"}, + []byte(`a,b +1,2 +3,4`), + newline([]byte(`a.b +5.2 +3.4`)), + nil, + nil, + )) } diff --git a/internal/command/select.go b/internal/command/select.go index e954e846..50af95f8 100644 --- a/internal/command/select.go +++ b/internal/command/select.go @@ -26,6 +26,10 @@ func selectFlags(cmd *cobra.Command) { cmd.Flags().Bool("pretty", true, "Pretty print the output.") cmd.Flags().Bool("colour", false, "Print colourised output.") cmd.Flags().Bool("escape-html", false, "Escape HTML tags when writing output.") + cmd.Flags().String("csv-comma", ",", "Comma separator to use when working with csv files.") + cmd.Flags().String("csv-write-comma", "", "Comma separator used when writing csv files. Overrides csv-comma when writing.") + cmd.Flags().String("csv-comment", "", "Comma separator used when reading csv files.") + cmd.Flags().Bool("csv-crlf", false, "True to use CRLF when writing CSV files.") _ = cmd.MarkFlagFilename("file") } @@ -38,12 +42,18 @@ func selectRunE(cmd *cobra.Command, args []string) error { prettyPrintFlag, _ := cmd.Flags().GetBool("pretty") colourFlag, _ := cmd.Flags().GetBool("colour") escapeHTMLFlag, _ := cmd.Flags().GetBool("escape-html") + csvComma, _ := cmd.Flags().GetString("csv-comma") + csvWriteComma, _ := cmd.Flags().GetString("csv-write-comma") + csvComment, _ := cmd.Flags().GetString("csv-comment") + csvCRLF, _ := cmd.Flags().GetBool("csv-crlf") opts := &selectOptions{ Read: &readOptions{ - Reader: nil, - Parser: readParserFlag, - FilePath: fileFlag, + Reader: nil, + Parser: readParserFlag, + FilePath: fileFlag, + CsvComma: csvComma, + CsvComment: csvComment, }, Write: &writeOptions{ Writer: nil, @@ -52,6 +62,8 @@ func selectRunE(cmd *cobra.Command, args []string) error { PrettyPrint: prettyPrintFlag, Colourise: colourFlag, EscapeHTML: escapeHTMLFlag, + CsvComma: csvWriteComma, + CsvUseCRLF: csvCRLF, }, Selector: selectorFlag, } diff --git a/internal/command/select_test.go b/internal/command/select_test.go index 95602e1e..332ac9b4 100644 --- a/internal/command/select_test.go +++ b/internal/command/select_test.go @@ -300,4 +300,40 @@ d,e,f`)), nil, )) + t.Run("CSV custom separator", runTest( + []string{"-r", "csv", "-w", "csv", "--csv-comma", "."}, + []byte(`A.B.C +a.b.c +d.e.f`), + newline([]byte(`A.B.C +a.b.c +d.e.f`)), + nil, + nil, + )) + + t.Run("CSV change separator", runTest( + []string{"-r", "csv", "-w", "csv", "--csv-comma", ".", "--csv-write-comma", ","}, + []byte(`A.B.C +a.b.c +d.e.f`), + newline([]byte(`A,B,C +a,b,c +d,e,f`)), + nil, + nil, + )) + + t.Run("CSV change from default separator", runTest( + []string{"-r", "csv", "-w", "csv", "--csv-write-comma", "."}, + []byte(`A,B,C +a,b,c +d,e,f`), + newline([]byte(`A.B.C +a.b.c +d.e.f`)), + nil, + nil, + )) + } diff --git a/storage/csv.go b/storage/csv.go index cee683d9..7265a001 100644 --- a/storage/csv.go +++ b/storage/csv.go @@ -27,12 +27,26 @@ type CSVDocument struct { } // FromBytes returns some data that is represented by the given bytes. -func (p *CSVParser) FromBytes(byteData []byte) (dasel.Value, error) { +func (p *CSVParser) FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error) { if byteData == nil { return dasel.Value{}, fmt.Errorf("could not read csv file: no data") } reader := csv.NewReader(bytes.NewBuffer(byteData)) + + for _, o := range options { + switch o.Key { + case OptionCSVComma: + if value, ok := o.Value.(rune); ok { + reader.Comma = value + } + case OptionCSVComment: + if value, ok := o.Value.(rune); ok { + reader.Comment = value + } + } + } + res := make([]*dencoding.Map, 0) records, err := reader.ReadAll() if err != nil { @@ -158,6 +172,19 @@ func (p *CSVParser) ToBytes(value dasel.Value, options ...ReadWriteOption) ([]by buffer := new(bytes.Buffer) writer := csv.NewWriter(buffer) + for _, o := range options { + switch o.Key { + case OptionCSVComma: + if value, ok := o.Value.(rune); ok { + writer.Comma = value + } + case OptionCSVComment: + if value, ok := o.Value.(bool); ok { + writer.UseCRLF = value + } + } + } + // Allow for multi document output by just appending documents on the end of each other. // This is really only supported so as we have nicer output when converting to CSV from // other multi-document formats. diff --git a/storage/json.go b/storage/json.go index 8e4c1101..115a8851 100644 --- a/storage/json.go +++ b/storage/json.go @@ -18,7 +18,7 @@ type JSONParser struct { } // FromBytes returns some data that is represented by the given bytes. -func (p *JSONParser) FromBytes(byteData []byte) (dasel.Value, error) { +func (p *JSONParser) FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error) { res := make([]any, 0) decoder := dencoding.NewJSONDecoder(bytes.NewReader(byteData)) diff --git a/storage/option.go b/storage/option.go index 9fa71d73..4a625ff6 100644 --- a/storage/option.go +++ b/storage/option.go @@ -32,6 +32,30 @@ func EscapeHTMLOption(enabled bool) ReadWriteOption { } } +// CsvCommaOption returns an option that modifies the separator character for CSV files. +func CsvCommaOption(comma rune) ReadWriteOption { + return ReadWriteOption{ + Key: OptionCSVComma, + Value: comma, + } +} + +// CsvCommentOption returns an option that modifies the comment character for CSV files. +func CsvCommentOption(comma rune) ReadWriteOption { + return ReadWriteOption{ + Key: OptionCSVComment, + Value: comma, + } +} + +// CsvUseCRLFOption returns an option that modifies the comment character for CSV files. +func CsvUseCRLFOption(enabled bool) ReadWriteOption { + return ReadWriteOption{ + Key: OptionCSVUseCRLF, + Value: enabled, + } +} + // OptionKey is a defined type for keys within a ReadWriteOption. type OptionKey string @@ -44,6 +68,12 @@ const ( OptionColourise OptionKey = "colourise" // OptionEscapeHTML is the key used with EscapeHTMLOption. OptionEscapeHTML OptionKey = "escapeHtml" + // OptionCSVComma is the key used with CsvCommaOption. + OptionCSVComma OptionKey = "csvComma" + // OptionCSVComment is the key used with CsvCommentOption. + OptionCSVComment OptionKey = "csvComment" + // OptionCSVUseCRLF is the key used with CsvUseCRLFOption. + OptionCSVUseCRLF OptionKey = "csvUseCRLF" ) // ReadWriteOption is an option to be used when writing. diff --git a/storage/parser.go b/storage/parser.go index 2140eed3..311e9181 100644 --- a/storage/parser.go +++ b/storage/parser.go @@ -50,7 +50,7 @@ func (e UnknownParserErr) Error() string { // ReadParser can be used to convert bytes to data. type ReadParser interface { // FromBytes returns some data that is represented by the given bytes. - FromBytes(byteData []byte) (dasel.Value, error) + FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error) } // WriteParser can be used to convert data to bytes. @@ -104,21 +104,21 @@ func NewWriteParserFromString(parser string) (WriteParser, error) { } // LoadFromFile loads data from the given file. -func LoadFromFile(filename string, p ReadParser) (dasel.Value, error) { +func LoadFromFile(filename string, p ReadParser, options ...ReadWriteOption) (dasel.Value, error) { f, err := os.Open(filename) if err != nil { return dasel.Value{}, fmt.Errorf("could not open file: %w", err) } - return Load(p, f) + return Load(p, f, options...) } // Load loads data from the given io.Reader. -func Load(p ReadParser, reader io.Reader) (dasel.Value, error) { +func Load(p ReadParser, reader io.Reader, options ...ReadWriteOption) (dasel.Value, error) { byteData, err := io.ReadAll(reader) if err != nil { return dasel.Value{}, fmt.Errorf("could not read data: %w", err) } - return p.FromBytes(byteData) + return p.FromBytes(byteData, options...) } // Write writes the value to the given io.Writer. diff --git a/storage/plain.go b/storage/plain.go index 4cbf8136..a0d1f333 100644 --- a/storage/plain.go +++ b/storage/plain.go @@ -18,7 +18,7 @@ type PlainParser struct { var ErrPlainParserNotImplemented = fmt.Errorf("PlainParser.FromBytes not implemented") // FromBytes returns some data that is represented by the given bytes. -func (p *PlainParser) FromBytes(byteData []byte) (dasel.Value, error) { +func (p *PlainParser) FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error) { return dasel.Value{}, ErrPlainParserNotImplemented } diff --git a/storage/toml.go b/storage/toml.go index c545cb3d..14266b47 100644 --- a/storage/toml.go +++ b/storage/toml.go @@ -18,7 +18,7 @@ type TOMLParser struct { } // FromBytes returns some data that is represented by the given bytes. -func (p *TOMLParser) FromBytes(byteData []byte) (dasel.Value, error) { +func (p *TOMLParser) FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error) { res := make([]interface{}, 0) decoder := dencoding.NewTOMLDecoder(bytes.NewReader(byteData)) diff --git a/storage/xml.go b/storage/xml.go index 61a72e5c..34b98491 100644 --- a/storage/xml.go +++ b/storage/xml.go @@ -27,7 +27,7 @@ type XMLParser struct { } // FromBytes returns some data that is represented by the given bytes. -func (p *XMLParser) FromBytes(byteData []byte) (dasel.Value, error) { +func (p *XMLParser) FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error) { if byteData == nil { return dasel.Value{}, fmt.Errorf("cannot parse nil xml data") } diff --git a/storage/yaml.go b/storage/yaml.go index f6a8cd12..1b049b4f 100644 --- a/storage/yaml.go +++ b/storage/yaml.go @@ -19,7 +19,7 @@ type YAMLParser struct { } // FromBytes returns some data that is represented by the given bytes. -func (p *YAMLParser) FromBytes(byteData []byte) (dasel.Value, error) { +func (p *YAMLParser) FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error) { res := make([]interface{}, 0) decoder := dencoding.NewYAMLDecoder(bytes.NewReader(byteData))