Merge pull request #367 from TomWright/csv-flags

Add CSV utilities

CHANGELOG.md

@@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - `orDefault()` function. [See docs](https://daseldocs.tomwright.me/functions/ordefault)
+- `--csv-comma` flag to change the csv separator.
+- `--csv-write-comma` flag to change the csv separator specifically for writes.
+- `--csv-comment` flag to change the csv comment character.
+- `--csv-crlf` flag to enable or disable CRLF output when working with csv files.
 
 ### Fixed
 

internal/command/delete.go

@@ -27,6 +27,10 @@ func deleteFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("pretty", true, "Pretty print the output.")
 	cmd.Flags().Bool("colour", false, "Print colourised output.")
 	cmd.Flags().Bool("escape-html", false, "Escape HTML tags when writing output.")
+	cmd.Flags().String("csv-comma", ",", "Comma separator to use when working with csv files.")
+	cmd.Flags().String("csv-write-comma", "", "Comma separator used when writing csv files. Overrides csv-comma when writing.")
+	cmd.Flags().String("csv-comment", "", "Comma separator used when reading csv files.")
+	cmd.Flags().Bool("csv-crlf", false, "True to use CRLF when writing CSV files.")
 
 	_ = cmd.MarkFlagFilename("file")
 }
@@ -40,12 +44,18 @@ func deleteRunE(cmd *cobra.Command, args []string) error {
 	colourFlag, _ := cmd.Flags().GetBool("colour")
 	escapeHTMLFlag, _ := cmd.Flags().GetBool("escape-html")
 	outFlag, _ := cmd.Flags().GetString("out")
+	csvComma, _ := cmd.Flags().GetString("csv-comma")
+	csvWriteComma, _ := cmd.Flags().GetString("csv-write-comma")
+	csvComment, _ := cmd.Flags().GetString("csv-comment")
+	csvCRLF, _ := cmd.Flags().GetBool("csv-crlf")
 
 	opts := &deleteOptions{
 		Read: &readOptions{
-			Reader:   nil,
-			Parser:   readParserFlag,
-			FilePath: fileFlag,
+			Reader:     nil,
+			Parser:     readParserFlag,
+			FilePath:   fileFlag,
+			CsvComma:   csvComma,
+			CsvComment: csvComment,
 		},
 		Write: &writeOptions{
 			Writer:      nil,
@@ -54,6 +64,8 @@ func deleteRunE(cmd *cobra.Command, args []string) error {
 			PrettyPrint: prettyPrintFlag,
 			Colourise:   colourFlag,
 			EscapeHTML:  escapeHTMLFlag,
+			CsvComma:    csvWriteComma,
+			CsvUseCRLF:  csvCRLF,
 		},
 		Selector: selectorFlag,
 	}

internal/command/options.go

@@ -16,6 +16,10 @@ type readOptions struct {
 	Parser string
 	// FilePath is the path to the source file.
 	FilePath string
+	// CsvComma is the comma character used when reading CSV files.
+	CsvComma string
+	// CsvComment is the comment character used when reading CSV files.
+	CsvComment string
 }
 
 func (o *readOptions) readFromStdin() bool {
@@ -49,6 +53,14 @@ func (o *readOptions) rootValue(cmd *cobra.Command) (dasel.Value, error) {
 		return dasel.Value{}, fmt.Errorf("could not get read parser: %w", err)
 	}
 
+	options := make([]storage.ReadWriteOption, 0)
+	if o.CsvComma != "" {
+		options = append(options, storage.CsvCommaOption([]rune(o.CsvComma)[0]))
+	}
+	if o.CsvComment != "" {
+		options = append(options, storage.CsvCommentOption([]rune(o.CsvComment)[0]))
+	}
+
 	reader := o.Reader
 	if reader == nil {
 		if o.readFromStdin() {
@@ -63,7 +75,7 @@ func (o *readOptions) rootValue(cmd *cobra.Command) (dasel.Value, error) {
 		}
 	}
 
-	return storage.Load(parser, reader)
+	return storage.Load(parser, reader, options...)
 }
 
 type writeOptions struct {
@@ -77,6 +89,11 @@ type writeOptions struct {
 	PrettyPrint bool
 	Colourise   bool
 	EscapeHTML  bool
+
+	// CsvComma is the comma character used when writing CSV files.
+	CsvComma string
+	// CsvUseCRLF determines whether CRLF is used when writing CSV files.
+	CsvUseCRLF bool
 }
 
 func (o *writeOptions) writeToStdout() bool {
@@ -121,6 +138,15 @@ func (o *writeOptions) writeValues(cmd *cobra.Command, readOptions *readOptions,
 		storage.ColouriseOption(o.Colourise),
 		storage.EscapeHTMLOption(o.EscapeHTML),
 		storage.PrettyPrintOption(o.PrettyPrint),
+		storage.CsvUseCRLFOption(o.CsvUseCRLF),
+	}
+
+	if o.CsvComma == "" && readOptions.CsvComma != "" {
+		o.CsvComma = readOptions.CsvComma
+	}
+
+	if o.CsvComma != "" {
+		options = append(options, storage.CsvCommaOption([]rune(o.CsvComma)[0]))
 	}
 
 	writer := o.Writer

internal/command/put.go

@@ -32,6 +32,10 @@ func putFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("pretty", true, "Pretty print the output.")
 	cmd.Flags().Bool("colour", false, "Print colourised output.")
 	cmd.Flags().Bool("escape-html", false, "Escape HTML tags when writing output.")
+	cmd.Flags().String("csv-comma", ",", "Comma separator to use when working with csv files.")
+	cmd.Flags().String("csv-write-comma", "", "Comma separator used when writing csv files. Overrides csv-comma when writing.")
+	cmd.Flags().String("csv-comment", "", "Comma separator used when reading csv files.")
+	cmd.Flags().Bool("csv-crlf", false, "True to use CRLF when writing CSV files.")
 
 	_ = cmd.MarkFlagFilename("file")
 }
@@ -47,12 +51,18 @@ func putRunE(cmd *cobra.Command, args []string) error {
 	colourFlag, _ := cmd.Flags().GetBool("colour")
 	escapeHTMLFlag, _ := cmd.Flags().GetBool("escape-html")
 	outFlag, _ := cmd.Flags().GetString("out")
+	csvComma, _ := cmd.Flags().GetString("csv-comma")
+	csvWriteComma, _ := cmd.Flags().GetString("csv-write-comma")
+	csvComment, _ := cmd.Flags().GetString("csv-comment")
+	csvCRLF, _ := cmd.Flags().GetBool("csv-crlf")
 
 	opts := &putOptions{
 		Read: &readOptions{
-			Reader:   nil,
-			Parser:   readParserFlag,
-			FilePath: fileFlag,
+			Reader:     nil,
+			Parser:     readParserFlag,
+			FilePath:   fileFlag,
+			CsvComma:   csvComma,
+			CsvComment: csvComment,
 		},
 		Write: &writeOptions{
 			Writer:      nil,
@@ -61,6 +71,8 @@ func putRunE(cmd *cobra.Command, args []string) error {
 			PrettyPrint: prettyPrintFlag,
 			Colourise:   colourFlag,
 			EscapeHTML:  escapeHTMLFlag,
+			CsvComma:    csvWriteComma,
+			CsvUseCRLF:  csvCRLF,
 		},
 		Selector:  selectorFlag,
 		ValueType: typeFlag,

internal/command/put_test.go

@@ -188,4 +188,16 @@ func TestPutCommand(t *testing.T) {
 		nil,
 		nil,
 	))
+
+	t.Run("CsvChangeSeparator", runTest(
+		[]string{"put", "-r", "csv", "-t", "int", "-v", "5", "--csv-write-comma", ".", "[0].a"},
+		[]byte(`a,b
+1,2
+3,4`),
+		newline([]byte(`a.b
+5.2
+3.4`)),
+		nil,
+		nil,
+	))
 }

internal/command/select.go

@@ -26,6 +26,10 @@ func selectFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("pretty", true, "Pretty print the output.")
 	cmd.Flags().Bool("colour", false, "Print colourised output.")
 	cmd.Flags().Bool("escape-html", false, "Escape HTML tags when writing output.")
+	cmd.Flags().String("csv-comma", ",", "Comma separator to use when working with csv files.")
+	cmd.Flags().String("csv-write-comma", "", "Comma separator used when writing csv files. Overrides csv-comma when writing.")
+	cmd.Flags().String("csv-comment", "", "Comma separator used when reading csv files.")
+	cmd.Flags().Bool("csv-crlf", false, "True to use CRLF when writing CSV files.")
 
 	_ = cmd.MarkFlagFilename("file")
 }
@@ -38,12 +42,18 @@ func selectRunE(cmd *cobra.Command, args []string) error {
 	prettyPrintFlag, _ := cmd.Flags().GetBool("pretty")
 	colourFlag, _ := cmd.Flags().GetBool("colour")
 	escapeHTMLFlag, _ := cmd.Flags().GetBool("escape-html")
+	csvComma, _ := cmd.Flags().GetString("csv-comma")
+	csvWriteComma, _ := cmd.Flags().GetString("csv-write-comma")
+	csvComment, _ := cmd.Flags().GetString("csv-comment")
+	csvCRLF, _ := cmd.Flags().GetBool("csv-crlf")
 
 	opts := &selectOptions{
 		Read: &readOptions{
-			Reader:   nil,
-			Parser:   readParserFlag,
-			FilePath: fileFlag,
+			Reader:     nil,
+			Parser:     readParserFlag,
+			FilePath:   fileFlag,
+			CsvComma:   csvComma,
+			CsvComment: csvComment,
 		},
 		Write: &writeOptions{
 			Writer:      nil,
@@ -52,6 +62,8 @@ func selectRunE(cmd *cobra.Command, args []string) error {
 			PrettyPrint: prettyPrintFlag,
 			Colourise:   colourFlag,
 			EscapeHTML:  escapeHTMLFlag,
+			CsvComma:    csvWriteComma,
+			CsvUseCRLF:  csvCRLF,
 		},
 		Selector: selectorFlag,
 	}

internal/command/select_test.go

@@ -300,4 +300,40 @@ d,e,f`)),
 		nil,
 	))
 
+	t.Run("CSV custom separator", runTest(
+		[]string{"-r", "csv", "-w", "csv", "--csv-comma", "."},
+		[]byte(`A.B.C
+a.b.c
+d.e.f`),
+		newline([]byte(`A.B.C
+a.b.c
+d.e.f`)),
+		nil,
+		nil,
+	))
+
+	t.Run("CSV change separator", runTest(
+		[]string{"-r", "csv", "-w", "csv", "--csv-comma", ".", "--csv-write-comma", ","},
+		[]byte(`A.B.C
+a.b.c
+d.e.f`),
+		newline([]byte(`A,B,C
+a,b,c
+d,e,f`)),
+		nil,
+		nil,
+	))
+
+	t.Run("CSV change from default separator", runTest(
+		[]string{"-r", "csv", "-w", "csv", "--csv-write-comma", "."},
+		[]byte(`A,B,C
+a,b,c
+d,e,f`),
+		newline([]byte(`A.B.C
+a.b.c
+d.e.f`)),
+		nil,
+		nil,
+	))
+
 }

storage/csv.go

@@ -27,12 +27,26 @@ type CSVDocument struct {
 }
 
 // FromBytes returns some data that is represented by the given bytes.
-func (p *CSVParser) FromBytes(byteData []byte) (dasel.Value, error) {
+func (p *CSVParser) FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error) {
 	if byteData == nil {
 		return dasel.Value{}, fmt.Errorf("could not read csv file: no data")
 	}
 
 	reader := csv.NewReader(bytes.NewBuffer(byteData))
+
+	for _, o := range options {
+		switch o.Key {
+		case OptionCSVComma:
+			if value, ok := o.Value.(rune); ok {
+				reader.Comma = value
+			}
+		case OptionCSVComment:
+			if value, ok := o.Value.(rune); ok {
+				reader.Comment = value
+			}
+		}
+	}
+
 	res := make([]*dencoding.Map, 0)
 	records, err := reader.ReadAll()
 	if err != nil {
@@ -158,6 +172,19 @@ func (p *CSVParser) ToBytes(value dasel.Value, options ...ReadWriteOption) ([]by
 	buffer := new(bytes.Buffer)
 	writer := csv.NewWriter(buffer)
 
+	for _, o := range options {
+		switch o.Key {
+		case OptionCSVComma:
+			if value, ok := o.Value.(rune); ok {
+				writer.Comma = value
+			}
+		case OptionCSVComment:
+			if value, ok := o.Value.(bool); ok {
+				writer.UseCRLF = value
+			}
+		}
+	}
+
 	// Allow for multi document output by just appending documents on the end of each other.
 	// This is really only supported so as we have nicer output when converting to CSV from
 	// other multi-document formats.

storage/json.go

@@ -18,7 +18,7 @@ type JSONParser struct {
 }
 
 // FromBytes returns some data that is represented by the given bytes.
-func (p *JSONParser) FromBytes(byteData []byte) (dasel.Value, error) {
+func (p *JSONParser) FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error) {
 	res := make([]any, 0)
 
 	decoder := dencoding.NewJSONDecoder(bytes.NewReader(byteData))

storage/option.go

@@ -32,6 +32,30 @@ func EscapeHTMLOption(enabled bool) ReadWriteOption {
 	}
 }
 
+// CsvCommaOption returns an option that modifies the separator character for CSV files.
+func CsvCommaOption(comma rune) ReadWriteOption {
+	return ReadWriteOption{
+		Key:   OptionCSVComma,
+		Value: comma,
+	}
+}
+
+// CsvCommentOption returns an option that modifies the comment character for CSV files.
+func CsvCommentOption(comma rune) ReadWriteOption {
+	return ReadWriteOption{
+		Key:   OptionCSVComment,
+		Value: comma,
+	}
+}
+
+// CsvUseCRLFOption returns an option that modifies the comment character for CSV files.
+func CsvUseCRLFOption(enabled bool) ReadWriteOption {
+	return ReadWriteOption{
+		Key:   OptionCSVUseCRLF,
+		Value: enabled,
+	}
+}
+
 // OptionKey is a defined type for keys within a ReadWriteOption.
 type OptionKey string
 
@@ -44,6 +68,12 @@ const (
 	OptionColourise OptionKey = "colourise"
 	// OptionEscapeHTML is the key used with EscapeHTMLOption.
 	OptionEscapeHTML OptionKey = "escapeHtml"
+	// OptionCSVComma is the key used with CsvCommaOption.
+	OptionCSVComma OptionKey = "csvComma"
+	// OptionCSVComment is the key used with CsvCommentOption.
+	OptionCSVComment OptionKey = "csvComment"
+	// OptionCSVUseCRLF is the key used with CsvUseCRLFOption.
+	OptionCSVUseCRLF OptionKey = "csvUseCRLF"
 )
 
 // ReadWriteOption is an option to be used when writing.

storage/parser.go

@@ -50,7 +50,7 @@ func (e UnknownParserErr) Error() string {
 // ReadParser can be used to convert bytes to data.
 type ReadParser interface {
 	// FromBytes returns some data that is represented by the given bytes.
-	FromBytes(byteData []byte) (dasel.Value, error)
+	FromBytes(byteData []byte, options ...ReadWriteOption) (dasel.Value, error)
 }
 
 // WriteParser can be used to convert data to bytes.
@@ -104,21 +104,21 @@ func NewWriteParserFromString(parser string) (WriteParser, error) {
 }
 
 // LoadFromFile loads data from the given file.
-func LoadFromFile(filename string, p ReadParser) (dasel.Value, error) {
+func LoadFromFile(filename string, p ReadParser, options ...ReadWriteOption) (dasel.Value, error) {
 	f, err := os.Open(filename)
 	if err != nil {
 		return dasel.Value{}, fmt.Errorf("could not open file: %w", err)
 	}
-	return Load(p, f)
+	return Load(p, f, options...)
 }
 
 // Load loads data from the given io.Reader.
-func Load(p ReadParser, reader io.Reader) (dasel.Value, error) {
+func Load(p ReadParser, reader io.Reader, options ...ReadWriteOption) (dasel.Value, error) {
 	byteData, err := io.ReadAll(reader)
 	if err != nil {
 		return dasel.Value{}, fmt.Errorf("could not read data: %w", err)
 	}
-	return p.FromBytes(byteData)
+	return p.FromBytes(byteData, options...)
 }
 
 // Write writes the value to the given io.Writer.