From f840dbd2d04bd73629494bced117ff5a7f9a0d8d Mon Sep 17 00:00:00 2001 From: stuioco Date: Mon, 19 Aug 2024 15:45:34 +0100 Subject: [PATCH] Manipulate csv contents --- core/templating/datasource.go | 7 +- core/templating/template_helpers.go | 131 ++++++++++++++---- core/templating/templating.go | 3 + core/templating/templating_test.go | 54 ++++++++ .../keyconcepts/templating/templating.rst | 76 +++++++++- 5 files changed, 240 insertions(+), 31 deletions(-) diff --git a/core/templating/datasource.go b/core/templating/datasource.go index b5ac59f60..0dcaf9f0e 100644 --- a/core/templating/datasource.go +++ b/core/templating/datasource.go @@ -2,15 +2,18 @@ package templating import ( "encoding/csv" + "strings" + "sync" + v2 "github.com/SpectoLabs/hoverfly/core/handlers/v2" log "github.com/sirupsen/logrus" - "strings" ) type DataSource struct { SourceType string Name string Data [][]string + mu sync.Mutex } func NewCsvDataSource(fileName, fileContent string) (*DataSource, error) { @@ -21,7 +24,7 @@ func NewCsvDataSource(fileName, fileContent string) (*DataSource, error) { return nil, err } - return &DataSource{"csv", fileName, records}, nil + return &DataSource{"csv", fileName, records, sync.Mutex{}}, nil } func (dataSource DataSource) GetDataSourceView() (v2.CSVDataSourceView, error) { diff --git a/core/templating/template_helpers.go b/core/templating/template_helpers.go index 36d8706bf..28df03453 100644 --- a/core/templating/template_helpers.go +++ b/core/templating/template_helpers.go @@ -213,39 +213,38 @@ func (t templateHelpers) faker(fakerType string) []reflect.Value { } func (t templateHelpers) fetchSingleFieldCsv(dataSourceName, searchFieldName, searchFieldValue, returnFieldName string, options *raymond.Options) string { - templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] - if exists { - searchIndex, err := getHeaderIndex(source.Data, searchFieldName) - if err != nil { - log.Error(err) - getEvaluationString("csv", options) - } - returnIndex, err := getHeaderIndex(source.Data, returnFieldName) - if err != nil { - log.Error(err) - return getEvaluationString("csv", options) - } - - var fallbackString string - searchFieldValue := getSearchFieldValue(options, searchFieldValue) - for i := 1; i < len(source.Data); i++ { - record := source.Data[i] - if strings.ToLower(record[searchIndex]) == strings.ToLower(searchFieldValue) { - return record[returnIndex] - } else if record[searchIndex] == "*" { - fallbackString = record[returnIndex] - } - } - - if fallbackString != "" { - return fallbackString + if !exists { + log.Debug("could not find datasource " + dataSourceName) + return getEvaluationString("csv", options) + } + source.mu.Lock() + defer source.mu.Unlock() + searchIndex, err := getHeaderIndex(source.Data, searchFieldName) + if err != nil { + log.Error(err) + return getEvaluationString("csv", options) + } + returnIndex, err := getHeaderIndex(source.Data, returnFieldName) + if err != nil { + log.Error(err) + return getEvaluationString("csv", options) + } + searchValue := getSearchFieldValue(options, searchFieldValue) + var fallbackString string + for i := 1; i < len(source.Data); i++ { + record := source.Data[i] + if strings.ToLower(record[searchIndex]) == strings.ToLower(searchValue) { + return record[returnIndex] + } else if record[searchIndex] == "*" { + fallbackString = record[returnIndex] } - + } + if fallbackString != "" { + return fallbackString } return getEvaluationString("csv", options) - } func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchFieldName string, searchFieldValue string) []map[string]string { @@ -259,6 +258,8 @@ func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchField log.Debug("no data available in datasource " + dataSourceName) return []map[string]string{} } + source.mu.Lock() + defer source.mu.Unlock() headers := source.Data[0] fieldIndex := -1 for i, header := range headers { @@ -291,6 +292,8 @@ func (t templateHelpers) csvAsArray(dataSourceName string) [][]string { templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] if exists { + source.mu.Lock() + defer source.mu.Unlock() return source.Data } else { log.Debug("could not find datasource " + dataSourceName) @@ -305,6 +308,8 @@ func (t templateHelpers) csvAsMap(dataSourceName string) []map[string]string { log.Debug("could not find datasource " + dataSourceName) return []map[string]string{} } + source.mu.Lock() + defer source.mu.Unlock() if len(source.Data) < 1 { log.Debug("no data available in datasource " + dataSourceName) return []map[string]string{} @@ -323,6 +328,76 @@ func (t templateHelpers) csvAsMap(dataSourceName string) []map[string]string { return result } +func (t templateHelpers) csvAddRow(dataSourceName string, newRow []string) string { + templateDataSources := t.TemplateDataSource.DataSources + source, exists := templateDataSources[dataSourceName] + if exists { + source.mu.Lock() + defer source.mu.Unlock() + source.Data = append(source.Data, newRow) + } else { + log.Debug("could not find datasource " + dataSourceName) + } + return "" +} + +func (t templateHelpers) csvDeleteRows(dataSourceName, searchFieldName, searchFieldValue string, output bool) string { + templateDataSources := t.TemplateDataSource.DataSources + source, exists := templateDataSources[dataSourceName] + if !exists { + log.Debug("could not find datasource " + dataSourceName) + return "" + } + source.mu.Lock() + defer source.mu.Unlock() + if len(source.Data) == 0 { + log.Debug("datasource " + dataSourceName + " is empty") + return "" + } + header := source.Data[0] + fieldIndex := -1 + for i, fieldName := range header { + if fieldName == searchFieldName { + fieldIndex = i + break + } + } + if fieldIndex == -1 { + log.Debug("could not find field name " + searchFieldName + " in datasource " + dataSourceName) + return "" + } + filteredData := [][]string{header} + rowsDeleted := 0 + for _, row := range source.Data[1:] { + if row[fieldIndex] != searchFieldValue { + filteredData = append(filteredData, row) + } else { + rowsDeleted++ + } + } + source.Data = filteredData + if output { + return fmt.Sprintf("%d", rowsDeleted) + } + return "" +} + +func (t templateHelpers) csvCountRows(dataSourceName string) string { + templateDataSources := t.TemplateDataSource.DataSources + source, exists := templateDataSources[dataSourceName] + if !exists { + log.Debug("could not find datasource " + dataSourceName) + return "" + } + source.mu.Lock() + defer source.mu.Unlock() + if len(source.Data) == 0 { + return "0" + } + numRows := len(source.Data) - 1 // The number of rows is len(source.Data) - 1 (subtracting 1 for the header row) + return fmt.Sprintf("%d", numRows) +} + func (t templateHelpers) parseJournalBasedOnIndex(indexName, keyValue, dataSource, queryType, lookupQuery string, options *raymond.Options) interface{} { journalDetails := options.Value("Journal").(Journal) if journalEntry, err := getIndexEntry(journalDetails, indexName, keyValue); err == nil { diff --git a/core/templating/templating.go b/core/templating/templating.go index 6a25ee4be..0e8e4abe8 100644 --- a/core/templating/templating.go +++ b/core/templating/templating.go @@ -104,6 +104,9 @@ func NewTemplator() *Templator { helperMethodMap["csvMatchingRows"] = t.fetchMatchingRowsCsv helperMethodMap["csvAsArray"] = t.csvAsArray helperMethodMap["csvAsMap"] = t.csvAsMap + helperMethodMap["csvAddRow"] = t.csvAddRow + helperMethodMap["csvDeleteRows"] = t.csvDeleteRows + helperMethodMap["csvCountRows"] = t.csvCountRows helperMethodMap["journal"] = t.parseJournalBasedOnIndex helperMethodMap["hasJournalKey"] = t.hasJournalKey helperMethodMap["setStatusCode"] = t.setStatusCode diff --git a/core/templating/templating_test.go b/core/templating/templating_test.go index 0e15567dc..27bffd305 100644 --- a/core/templating/templating_test.go +++ b/core/templating/templating_test.go @@ -140,6 +140,60 @@ func Test_ApplyTemplate_CsvAsMapMissingDataSource(t *testing.T) { // ------------------------------- +func Test_ApplyTemplate_CsvAddRow(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '99' false}}{{addToArray 'newMark' 'Violet' false}}{{addToArray 'newMark' '55' false}}{{csvAddRow 'test-csv1' (getArray 'newMark')}}{{csv 'test-csv1' 'id' '99' 'name'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`Violet`)) +} + +func Test_ApplyTemplate_CsvDeleteRows(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv1' 'id' '2' true}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`1`)) +} + +func Test_ApplyTemplate_CsvDeleteMissingDataset(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv99' 'id' '2' true}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + +func Test_ApplyTemplate_CsvDeleteMissingField(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv1' 'identity' '2' true}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + +func Test_ApplyTemplate_CsvCountRows(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv1'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`3`)) +} + +func Test_ApplyTemplate_CsvCountRowsMissingDataset(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv99'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + func Test_ApplyTemplate_EachBlockWithSplitTemplatingFunction(t *testing.T) { RegisterTestingT(t) diff --git a/docs/pages/keyconcepts/templating/templating.rst b/docs/pages/keyconcepts/templating/templating.rst index cb10f05e2..929659603 100644 --- a/docs/pages/keyconcepts/templating/templating.rst +++ b/docs/pages/keyconcepts/templating/templating.rst @@ -173,7 +173,12 @@ Fakers that require arguments are currently not supported. CSV Data Source ~~~~~~~~~~~~~~~ -You can query data from a CSV data source in a number of ways. +You can both query data from a CSV data source as well as manipulate data within a data source byadding to it and deleting from it. + +Reading from a CSV Data Source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can read data from a CSV data source in a number of ways. The most basic is to return the value of one field (selected-column) given a field name to search (column-name) and a value to search for in that field (query-value). Of course the query-value would normally be pulled from the request. @@ -328,7 +333,76 @@ Example: Start Hoverfly with a CSV data source (pets.csv) provided below. | | | 1002 dogs Teddy sold | +--------------------------+------------------------------------------------------------+-----------------------------------------+ +Adding data to a CSV Data Source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While the service is running you can add new rows of data into the data source. This is not peristent, it is only manipulated in memory +and so it only lasts for as long as the service is running. The rows are not actually written to the file. + +.. code:: json + + { + "body": "{\"name\": \"{{csvAddRow '(data-source-name)' (array-of-values)}}\"}" + } + + +To use this function you first need to construct an array containing the row of string values to store in the csv data source. +For example say you had a csv called pets with the columns id, category, name and status. + +1. You would first add each of the 4 values into an array of 4 items to match the number of columns: + +``{{ addToArray 'newPet' '2000' false }}`` +``{{ addToArray 'newPet' 'dogs' false }}`` +``{{ addToArray 'newPet' 'Violet' false }}`` +``{{ addToArray 'newPet' 'sold' false }}`` + +2. You then call the csvAddRow function to add the row into the csv data store: + +``{{csvAddRow 'pets' (getArray 'newPet') }}`` + + +Deleting data from a CSV Data Source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While the service is running you can delete rows of data from the data source. This is not peristent, it is only manipulated in memory +and so it only lasts for as long as the service is running. The rows are not actually deleted from the file. +.. code:: json + + { + "body": "{\"name\": \"{{csvDeleteRows '(data-source-name)' '(column-name)' '(query-value)' (output-result)}}\"}" + } + +To delete rows from the csv data source your specify the value that a specific column must have to be deleted. + +To delete all the pets where the category is cats from the pets csv data store: + +``{{ csvDeleteRows 'pets' 'category' 'cats' false }}`` + +Note that the last parameter of "output-result" is a boolean. It is not enclosed in quotes. The function will return the number of rows +affected which can either be suppressed by passing false, or passed into another function if you need to make logical decisions based on the number of +rows affected by passing in true. If csvDeleteRows is not enclosed within another function it will output the number of rows +deleted to the template. + +``{{#equal (csvDeleteRows 'pets' 'category' 'cats' true) '0'}}`` +`` {{ setStatusCode '404' }}`` +`` {"Message":"Error no cats found"}`` +``{{else}}`` +`` {{ setStatusCode '200' }}`` +`` {"Message":"All cats deleted"}`` +``{{/equal}}`` + + +Counting the rows in a CSV Data Source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can return the number of rows in a csv dataset. This will be 1 less than the number of rows as the first row contains the column names. + +.. code:: json + + { + "body": "{\"name\": \"{{csvCountRows '(data-source-name)'}}\"}" + } Journal Entry Data