Manipulate csv contents

core/templating/datasource.go

@@ -2,15 +2,18 @@ package templating
 
 import (
 	"encoding/csv"
+	"strings"
+	"sync"
+
 	v2 "github.com/SpectoLabs/hoverfly/core/handlers/v2"
 	log "github.com/sirupsen/logrus"
-	"strings"
 )
 
 type DataSource struct {
 	SourceType string
 	Name       string
 	Data       [][]string
+	mu         sync.Mutex
 }
 
 func NewCsvDataSource(fileName, fileContent string) (*DataSource, error) {
@@ -21,7 +24,7 @@ func NewCsvDataSource(fileName, fileContent string) (*DataSource, error) {
 		return nil, err
 	}
 
-	return &DataSource{"csv", fileName, records}, nil
+	return &DataSource{"csv", fileName, records, sync.Mutex{}}, nil
 }
 
 func (dataSource DataSource) GetDataSourceView() (v2.CSVDataSourceView, error) {

core/templating/template_helpers.go

@@ -213,39 +213,38 @@ func (t templateHelpers) faker(fakerType string) []reflect.Value {
 }
 
 func (t templateHelpers) fetchSingleFieldCsv(dataSourceName, searchFieldName, searchFieldValue, returnFieldName string, options *raymond.Options) string {
-
 	templateDataSources := t.TemplateDataSource.DataSources
 	source, exists := templateDataSources[dataSourceName]
-	if exists {
-		searchIndex, err := getHeaderIndex(source.Data, searchFieldName)
-		if err != nil {
-			log.Error(err)
-			getEvaluationString("csv", options)
-		}
-		returnIndex, err := getHeaderIndex(source.Data, returnFieldName)
-		if err != nil {
-			log.Error(err)
-			return getEvaluationString("csv", options)
-		}
-
-		var fallbackString string
-		searchFieldValue := getSearchFieldValue(options, searchFieldValue)
-		for i := 1; i < len(source.Data); i++ {
-			record := source.Data[i]
-			if strings.ToLower(record[searchIndex]) == strings.ToLower(searchFieldValue) {
-				return record[returnIndex]
-			} else if record[searchIndex] == "*" {
-				fallbackString = record[returnIndex]
-			}
-		}
-
-		if fallbackString != "" {
-			return fallbackString
+	if !exists {
+		log.Debug("could not find datasource " + dataSourceName)
+		return getEvaluationString("csv", options)
+	}
+	source.mu.Lock()
+	defer source.mu.Unlock()
+	searchIndex, err := getHeaderIndex(source.Data, searchFieldName)
+	if err != nil {
+		log.Error(err)
+		return getEvaluationString("csv", options)
+	}
+	returnIndex, err := getHeaderIndex(source.Data, returnFieldName)
+	if err != nil {
+		log.Error(err)
+		return getEvaluationString("csv", options)
+	}
+	searchValue := getSearchFieldValue(options, searchFieldValue)
+	var fallbackString string
+	for i := 1; i < len(source.Data); i++ {
+		record := source.Data[i]
+		if strings.ToLower(record[searchIndex]) == strings.ToLower(searchValue) {
+			return record[returnIndex]
+		} else if record[searchIndex] == "*" {
+			fallbackString = record[returnIndex]
 		}
-
+	}
+	if fallbackString != "" {
+		return fallbackString
 	}
 	return getEvaluationString("csv", options)
-
 }
 
 func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchFieldName string, searchFieldValue string) []map[string]string {
@@ -259,6 +258,8 @@ func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchField
 		log.Debug("no data available in datasource " + dataSourceName)
 		return []map[string]string{}
 	}
+	source.mu.Lock()
+	defer source.mu.Unlock()
 	headers := source.Data[0]
 	fieldIndex := -1
 	for i, header := range headers {
@@ -291,6 +292,8 @@ func (t templateHelpers) csvAsArray(dataSourceName string) [][]string {
 	templateDataSources := t.TemplateDataSource.DataSources
 	source, exists := templateDataSources[dataSourceName]
 	if exists {
+		source.mu.Lock()
+		defer source.mu.Unlock()
 		return source.Data
 	} else {
 		log.Debug("could not find datasource " + dataSourceName)
@@ -305,6 +308,8 @@ func (t templateHelpers) csvAsMap(dataSourceName string) []map[string]string {
 		log.Debug("could not find datasource " + dataSourceName)
 		return []map[string]string{}
 	}
+	source.mu.Lock()
+	defer source.mu.Unlock()
 	if len(source.Data) < 1 {
 		log.Debug("no data available in datasource " + dataSourceName)
 		return []map[string]string{}
@@ -323,6 +328,76 @@ func (t templateHelpers) csvAsMap(dataSourceName string) []map[string]string {
 	return result
 }
 
+func (t templateHelpers) csvAddRow(dataSourceName string, newRow []string) string {
+	templateDataSources := t.TemplateDataSource.DataSources
+	source, exists := templateDataSources[dataSourceName]
+	if exists {
+		source.mu.Lock()
+		defer source.mu.Unlock()
+		source.Data = append(source.Data, newRow)
+	} else {
+		log.Debug("could not find datasource " + dataSourceName)
+	}
+	return ""
+}
+
+func (t templateHelpers) csvDeleteRows(dataSourceName, searchFieldName, searchFieldValue string, output bool) string {
+	templateDataSources := t.TemplateDataSource.DataSources
+	source, exists := templateDataSources[dataSourceName]
+	if !exists {
+		log.Debug("could not find datasource " + dataSourceName)
+		return ""
+	}
+	source.mu.Lock()
+	defer source.mu.Unlock()
+	if len(source.Data) == 0 {
+		log.Debug("datasource " + dataSourceName + " is empty")
+		return ""
+	}
+	header := source.Data[0]
+	fieldIndex := -1
+	for i, fieldName := range header {
+		if fieldName == searchFieldName {
+			fieldIndex = i
+			break
+		}
+	}
+	if fieldIndex == -1 {
+		log.Debug("could not find field name " + searchFieldName + " in datasource " + dataSourceName)
+		return ""
+	}
+	filteredData := [][]string{header}
+	rowsDeleted := 0
+	for _, row := range source.Data[1:] {
+		if row[fieldIndex] != searchFieldValue {
+			filteredData = append(filteredData, row)
+		} else {
+			rowsDeleted++
+		}
+	}
+	source.Data = filteredData
+	if output {
+		return fmt.Sprintf("%d", rowsDeleted)
+	}
+	return ""
+}
+
+func (t templateHelpers) csvCountRows(dataSourceName string) string {
+	templateDataSources := t.TemplateDataSource.DataSources
+	source, exists := templateDataSources[dataSourceName]
+	if !exists {
+		log.Debug("could not find datasource " + dataSourceName)
+		return ""
+	}
+	source.mu.Lock()
+	defer source.mu.Unlock()
+	if len(source.Data) == 0 {
+		return "0"
+	}
+	numRows := len(source.Data) - 1 // The number of rows is len(source.Data) - 1 (subtracting 1 for the header row)
+	return fmt.Sprintf("%d", numRows)
+}
+
 func (t templateHelpers) parseJournalBasedOnIndex(indexName, keyValue, dataSource, queryType, lookupQuery string, options *raymond.Options) interface{} {
 	journalDetails := options.Value("Journal").(Journal)
 	if journalEntry, err := getIndexEntry(journalDetails, indexName, keyValue); err == nil {

core/templating/templating.go

@@ -104,6 +104,9 @@ func NewTemplator() *Templator {
 	helperMethodMap["csvMatchingRows"] = t.fetchMatchingRowsCsv
 	helperMethodMap["csvAsArray"] = t.csvAsArray
 	helperMethodMap["csvAsMap"] = t.csvAsMap
+	helperMethodMap["csvAddRow"] = t.csvAddRow
+	helperMethodMap["csvDeleteRows"] = t.csvDeleteRows
+	helperMethodMap["csvCountRows"] = t.csvCountRows
 	helperMethodMap["journal"] = t.parseJournalBasedOnIndex
 	helperMethodMap["hasJournalKey"] = t.hasJournalKey
 	helperMethodMap["setStatusCode"] = t.setStatusCode

core/templating/templating_test.go

@@ -140,6 +140,60 @@ func Test_ApplyTemplate_CsvAsMapMissingDataSource(t *testing.T) {
 
 // -------------------------------
 
+func Test_ApplyTemplate_CsvAddRow(t *testing.T) {
+	RegisterTestingT(t)
+
+	template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '99' false}}{{addToArray 'newMark' 'Violet' false}}{{addToArray 'newMark' '55' false}}{{csvAddRow 'test-csv1' (getArray 'newMark')}}{{csv 'test-csv1' 'id' '99' 'name'}}`)
+
+	Expect(err).To(BeNil())
+	Expect(template).To(Equal(`Violet`))
+}
+
+func Test_ApplyTemplate_CsvDeleteRows(t *testing.T) {
+	RegisterTestingT(t)
+
+	template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv1' 'id' '2' true}}`)
+
+	Expect(err).To(BeNil())
+	Expect(template).To(Equal(`1`))
+}
+
+func Test_ApplyTemplate_CsvDeleteMissingDataset(t *testing.T) {
+	RegisterTestingT(t)
+
+	template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv99' 'id' '2' true}}`)
+
+	Expect(err).To(BeNil())
+	Expect(template).To(Equal(``))
+}
+
+func Test_ApplyTemplate_CsvDeleteMissingField(t *testing.T) {
+	RegisterTestingT(t)
+
+	template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv1' 'identity' '2' true}}`)
+
+	Expect(err).To(BeNil())
+	Expect(template).To(Equal(``))
+}
+
+func Test_ApplyTemplate_CsvCountRows(t *testing.T) {
+	RegisterTestingT(t)
+
+	template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv1'}}`)
+
+	Expect(err).To(BeNil())
+	Expect(template).To(Equal(`3`))
+}
+
+func Test_ApplyTemplate_CsvCountRowsMissingDataset(t *testing.T) {
+	RegisterTestingT(t)
+
+	template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv99'}}`)
+
+	Expect(err).To(BeNil())
+	Expect(template).To(Equal(``))
+}
+
 func Test_ApplyTemplate_EachBlockWithSplitTemplatingFunction(t *testing.T) {
 	RegisterTestingT(t)
 

docs/pages/keyconcepts/templating/templating.rst

@@ -173,7 +173,12 @@ Fakers that require arguments are currently not supported.
 CSV Data Source
 ~~~~~~~~~~~~~~~
 
-You can query data from a CSV data source in a number of ways.
+You can both query data from a CSV data source as well as manipulate data within a data source byadding to it and deleting from it.
+
+Reading from a CSV Data Source
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can read data from a CSV data source in a number of ways.
 
 The most basic is to return the value of one field (selected-column) given a field name to search (column-name) and 
 a value to search for in that field (query-value). Of course the query-value would normally be pulled from the request.
@@ -328,7 +333,76 @@ Example: Start Hoverfly with a CSV data source (pets.csv) provided below.
 |                          |                                                            | 1002 dogs Teddy sold                    |
 +--------------------------+------------------------------------------------------------+-----------------------------------------+
 
+Adding data to a CSV Data Source
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+While the service is running you can add new rows of data into the data source. This is not peristent, it is only manipulated in memory
+and so it only lasts for as long as the service is running. The rows are not actually written to the file.
+
+.. code:: json
+
+    {
+        "body": "{\"name\": \"{{csvAddRow '(data-source-name)' (array-of-values)}}\"}"
+    }
+
+
+To use this function you first need to construct an array containing the row of string values to store in the csv data source.
+For example say you had a csv called pets with the columns id, category, name and status.
+
+1. You would first add each of the 4 values into an array of 4 items to match the number of columns:
+
+``{{ addToArray 'newPet' '2000' false }}``
+``{{ addToArray 'newPet' 'dogs' false }}``
+``{{ addToArray 'newPet' 'Violet' false }}``
+``{{ addToArray 'newPet' 'sold' false }}``
+
+2. You then call the csvAddRow function to add the row into the csv data store:
+
+``{{csvAddRow 'pets' (getArray 'newPet') }}``
+
+
+Deleting data from a CSV Data Source
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+While the service is running you can delete rows of data from the data source. This is not peristent, it is only manipulated in memory
+and so it only lasts for as long as the service is running. The rows are not actually deleted from the file.
 
+.. code:: json
+
+    {
+        "body": "{\"name\": \"{{csvDeleteRows '(data-source-name)' '(column-name)' '(query-value)' (output-result)}}\"}"
+    }
+
+To delete rows from the csv data source your specify the value that a specific column must have to be deleted.
+
+To delete all the pets where the category is cats from the pets csv data store:
+
+``{{ csvDeleteRows 'pets' 'category' 'cats' false }}``
+
+Note that the last parameter of "output-result" is a boolean. It is not enclosed in quotes. The function will return the number of rows
+affected which can either be suppressed by passing false, or passed into another function if you need to make logical decisions based on the number of 
+rows affected by passing in true. If csvDeleteRows is not enclosed within another function it will output the number of rows
+deleted to the template.
+
+``{{#equal (csvDeleteRows 'pets' 'category' 'cats' true) '0'}}``
+``    {{ setStatusCode '404' }}``
+``    {"Message":"Error no cats found"}``
+``{{else}}``
+``    {{ setStatusCode '200' }}``
+``    {"Message":"All cats deleted"}``
+``{{/equal}}``
+
+
+Counting the rows in a CSV Data Source
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can return the number of rows in a csv dataset. This will be 1 less than the number of rows as the first row contains the column names.
+
+.. code:: json
+
+    {
+        "body": "{\"name\": \"{{csvCountRows '(data-source-name)'}}\"}"
+    }
 
 
 Journal Entry Data