support comma in column value for service test csv data

legend-engine-test-runner-service/src/test/java/org/finos/legend/engine/testable/service/TestServiceTestSuite.java

@@ -3081,6 +3081,22 @@ public void testPassingRelationalWithParams()
         Assert.assertEquals(passed.testSuiteId, "testSuite1");
     }
 
+    @Test
+    public void testRelationalServiceWithCommmaInCSV()
+    {
+        // setup
+        List<TestResult> relationalTestResult = executeServiceTest("testable/relational/", "legend-testable-relational-model.pure", "legend-testable-relational-service-csv-test-data.pure","service::RelationalServiceWithCSV");
+        // Assertions
+        Assert.assertEquals(relationalTestResult.size(), 1);
+        TestResult testResult = relationalTestResult.get(0);
+        Assert.assertEquals(testResult.testable, "service::RelationalServiceWithCSV");
+        Assert.assertTrue(testResult instanceof TestExecuted);
+        Assert.assertEquals(TestExecutionStatus.PASS, ((TestExecuted) testResult).testExecutionStatus);
+        TestExecuted passed = (TestExecuted) testResult;
+        Assert.assertEquals(passed.atomicTestId, "test_1");
+        Assert.assertEquals(passed.testSuiteId, "testSuite_1");
+    }
+
     @Test
     public void testPassingRelationalWithEnumParams()
     {

legend-engine-test-runner-service/src/test/resources/testable/relational/legend-testable-relational-service-csv-test-data.pure

@@ -0,0 +1,54 @@
+###Service
+Service service::RelationalServiceWithCSV
+{
+  pattern: '/myService/{firm}';
+  documentation: '';
+  autoActivateUpdates: true;
+  execution: Single
+  {
+    query: |model::Person.all()->project([x|$x.firstName, x|$x.lastName], ['First Name', 'Last Name']);
+    mapping: execution::RelationalMapping;
+    runtime: execution::Runtime;
+  }
+  testSuites:
+  [
+    testSuite_1:
+    {
+      data:
+      [
+        connections:
+        [
+          connection_1:
+            Relational
+            #{
+              default.PersonTable:
+                'id,firm_id,firstName,lastName,employeeType\n'+
+                '1,1,I\'m John, "Doe, Jr",FTO\n'+
+                '2,1,Nicole,Smith,FTC\n'+
+                '3,2,Time,Smith,FTE\n';
+            }#
+        ]
+      ]
+      tests:
+      [
+        test_1:
+        {
+          serializationFormat: PURE_TDSOBJECT;
+          asserts:
+          [
+            assertion_1:
+              EqualToJson
+              #{
+                expected:
+                  ExternalFormat
+                  #{
+                    contentType: 'application/json';
+                    data: '[ {\n  "First Name" : "I\'m John",\n  "Last Name" : "\\"Doe, Jr\\""\n}, {\n  "First Name" : "Nicole",\n  "Last Name" : "Smith"\n}, {\n  "First Name" : "Time",\n  "Last Name" : "Smith"\n} ]';
+                  }#;
+              }#
+          ]
+        }
+      ]
+    }
+  ]
+}

legend-engine-test-runner-service/src/test/resources/testable/relational/legend-testable-relational-service-embeddedData.pure

@@ -23,7 +23,7 @@ Service service::SimpleRelationalPassWithSpecialEmbeddedData
           #{
             default.PersonTable:
               'id,firm_id,firstName,lastName\n' +
-              '1,1,John;\'",Doe\n' +
+              '1,1,"John;\'",Doe\n' +
               '2,1,Nicole,Smith\n' +
               '3,2,Time,Smith\n';
 
@@ -48,7 +48,7 @@ Service service::SimpleRelationalPassWithSpecialEmbeddedData
                   ExternalFormat
                   #{
                     contentType: 'application/json';
-                    data: '[{"Employees/First Name":"John;\'\\"","Employees/Last Name":"Doe","Legal Name":"Finos"},{"Employees/First Name":"Nicole","Employees/Last Name":"Smith","Legal Name":"Finos"},{"Employees/First Name":"Time","Employees/Last Name":"Smith","Legal Name":"Apple"}]\n';
+                    data: '[{"Employees/First Name":"\\"John;\'\\"","Employees/Last Name":"Doe","Legal Name":"Finos"},{"Employees/First Name":"Nicole","Employees/Last Name":"Smith","Legal Name":"Finos"},{"Employees/First Name":"Time","Employees/Last Name":"Smith","Legal Name":"Apple"}]\n';
                   }#;
               }#
           ]

legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure

@@ -115,10 +115,46 @@ function <<access.private>> meta::alloy::service::execution::splitWithEmptyValue
    let delimForEmptyCsvField = '|EMPTY|';
 
    $s->meta::alloy::service::execution::replaceWithEmptyValue($delimForEmptyCsvField)
-        ->split(',')
+        ->meta::alloy::service::execution::splitCsvRow(',')
         ->map(csv_val| if($csv_val == $delimForEmptyCsvField, | '' ,| $csv_val));
 }
 
+Class <<access.private>> meta::alloy::service::execution::SplitterState
+{
+   tokens:String[*];
+   inQuotes:Boolean[1];
+   isPrevEscapeChar: Boolean[1];
+   currentTokenChars:String[*];
+}
+
+function meta::alloy::service::execution::splitCsvRow(s: String[1], delimiter: String[1]): String[*]
+{
+  let charArray = chunk($s, 1);
+  let finalSplitterState = $charArray->fold({currentChar,splitterState |
+                  if ($currentChar == $delimiter,
+                    | if(!$splitterState.inQuotes,
+                      | ^$splitterState(tokens = $splitterState.tokens->add($splitterState.currentTokenChars->makeString()->trim()), inQuotes= false, isPrevEscapeChar = false, currentTokenChars=[]),
+                      | ^$splitterState(tokens = $splitterState.tokens, isPrevEscapeChar = false, currentTokenChars = $splitterState.currentTokenChars->add($currentChar))
+                      ),
+                    | if ($currentChar == '"',
+                        | if ($splitterState.isPrevEscapeChar,
+                        | ^$splitterState(isPrevEscapeChar = false, currentTokenChars = $splitterState.currentTokenChars->add($currentChar));,
+                        | if(!$splitterState.inQuotes,
+                            | ^$splitterState(inQuotes = true, currentTokenChars = $splitterState.currentTokenChars->add($currentChar)),
+                            | ^$splitterState(inQuotes = false, currentTokenChars = $splitterState.currentTokenChars->add($currentChar))
+                            )
+                           ),
+                        | if ( $currentChar == '\\',
+                           | ^$splitterState(isPrevEscapeChar = true, currentTokenChars = $splitterState.currentTokenChars->add($currentChar)),
+                           | ^$splitterState(isPrevEscapeChar = false, currentTokenChars = $splitterState.currentTokenChars->add($currentChar))
+                             )
+                       )
+                    )                            
+                  }, ^meta::alloy::service::execution::SplitterState(tokens=[], inQuotes= false, isPrevEscapeChar = false, currentTokenChars=[]));
+
+  if($finalSplitterState.currentTokenChars != [],|$finalSplitterState.tokens->add($finalSplitterState.currentTokenChars->makeString()->trim()), | $finalSplitterState.tokens);
+}
+
 function <<access.private>> meta::alloy::service::execution::replaceWithEmptyValue(s:String[1], delimForEmptyCsvField: String[1]) : String[1]
 {
    let news = if($s->endsWith(','), | $s + $delimForEmptyCsvField, |$s);

legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/DDL/testDDL.pure

@@ -112,4 +112,45 @@ function <<test.Test>>  meta::relational::tests::ddl::testSetupDataSqlGeneration
    'insert into personTable (ID,FIRSTNAME,LASTNAME,AGE,ADDRESSID,FIRMID,MANAGERID) values (1,\'Peter\',\'Smith\',23,1,1,2);'
    ];
   assertSameElements($sqls,$expectedSqls);
-}  
+}  
+
+function <<test.Test>>  meta::relational::tests::ddl::testSetupDataSqlGenerationWithColumnValueHasDelimiterAndQuotes():Boolean[1]
+{
+  let records ='default\n'+
+                'personTable\n'+
+                'id, firstName, lastName, age, addressId, firmId, managerId\n'+
+                '1, Peter,  "I\'m Smith, Jr",23,1,1,2';
+
+  let sqls = meta::alloy::service::execution::setUpDataSQLs($records, meta::relational::tests::dbInc, meta::relational::functions::sqlQueryToString::createDbConfig(DatabaseType.H2));
+
+  let expectedSqls= [
+   'Drop schema if exists productSchema cascade;',
+   'Create Schema productSchema;',
+   'Drop schema if exists default cascade;',
+   'Create Schema default;',
+   'Drop table if exists productSchema.productTable;',
+   'Create Table productSchema.productTable(ID INT NOT NULL,NAME VARCHAR(200) NULL, PRIMARY KEY(ID));',
+   'Drop table if exists personTable;',
+   'Create Table personTable(ID INT NOT NULL,FIRSTNAME VARCHAR(200) NULL,LASTNAME VARCHAR(200) NULL,AGE INT NULL,ADDRESSID INT NULL,FIRMID INT NULL,MANAGERID INT NULL, PRIMARY KEY(ID));',
+   'Drop table if exists PersonTableExtension;',
+   'Create Table PersonTableExtension(ID INT NOT NULL,FIRSTNAME VARCHAR(200) NULL,LASTNAME VARCHAR(200) NULL,AGE INT NULL,ADDRESSID INT NULL,FIRMID INT NULL,MANAGERID INT NULL,birthDate DATE NULL, PRIMARY KEY(ID));',
+   'Drop table if exists differentPersonTable;',
+   'Create Table differentPersonTable(ID INT NOT NULL,FIRSTNAME VARCHAR(200) NULL,LASTNAME VARCHAR(200) NULL,AGE INT NULL,ADDRESSID INT NULL,FIRMID INT NULL,MANAGERID INT NULL, PRIMARY KEY(ID));',
+   'Drop table if exists firmTable;',
+   'Create Table firmTable(ID INT NOT NULL,LEGALNAME VARCHAR(200) NULL,ADDRESSID INT NULL,CEOID INT NULL, PRIMARY KEY(ID));',
+   'Drop table if exists firmExtensionTable;',
+   'Create Table firmExtensionTable(firmId INT NOT NULL,legalName VARCHAR(200) NULL,establishedDate DATE NULL, PRIMARY KEY(firmId));',
+   'Drop table if exists otherFirmTable;',
+   'Create Table otherFirmTable(ID INT NOT NULL,LEGALNAME VARCHAR(200) NULL,ADDRESSID INT NULL, PRIMARY KEY(ID));',
+   'Drop table if exists addressTable;',
+   'Create Table addressTable(ID INT NOT NULL,TYPE INT NULL,NAME VARCHAR(200) NULL,STREET VARCHAR(100) NULL,COMMENTS VARCHAR(100) NULL, PRIMARY KEY(ID));',
+   'Drop table if exists locationTable;',
+   'Create Table locationTable(ID INT NOT NULL,PERSONID INT NULL,PLACE VARCHAR(200) NULL,date DATE NULL, PRIMARY KEY(ID));',
+   'Drop table if exists placeOfInterestTable;',
+   'Create Table placeOfInterestTable(ID INT NOT NULL,locationID INT NOT NULL,NAME VARCHAR(200) NULL, PRIMARY KEY(ID,locationID));',
+   'Drop table if exists validPersonTable;',
+   'Create Table validPersonTable(ID INT NOT NULL,FIRSTNAME VARCHAR(200) NULL,LASTNAME VARCHAR(200) NULL,AGE INT NULL,ADDRESSID INT NULL,FIRMID INT NULL,MANAGERID INT NULL, PRIMARY KEY(ID));',
+   'insert into personTable (ID,FIRSTNAME,LASTNAME,AGE,ADDRESSID,FIRMID,MANAGERID) values (1,\'Peter\',\'"I\'\'m Smith, Jr"\',23,1,1,2);'
+   ];
+  assertSameElements($sqls,$expectedSqls);
+}

legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/naming/naming.pure

@@ -116,7 +116,8 @@ function <<paramTest.BeforePackage>> meta::relational::tests::dbSpecificTests::s
   //  'insert into "test Table2"("Column&ID", "My#Data") values (3,\'Description A\');'
   //  ];
 
-   let records ='default\n'+
+   let records =
+               'default\n'+
                 'testTable\n'+
                 'ID, "Column Name with Space", "Column&A", "Column.ID"\n'+
                 '1,2,0,3\n'+
@@ -129,7 +130,7 @@ function <<paramTest.BeforePackage>> meta::relational::tests::dbSpecificTests::s
                  '---\n'+
                 'default\n'+
                 'testTable3\n'+
-                'ABCDEFGHIJKLMNOPQRSTUVWXYZ,"0123456789","!\"#$%&()*+-./:;<=>?@[\]^_`{|}",abcdefghijklmnopqrstuvwxzy\n'+
+                'ABCDEFGHIJKLMNOPQRSTUVWXYZ,"0123456789","!\\"#$%&()*+-./:;<=>?@[\]^_`{|}",abcdefghijklmnopqrstuvwxzy\n'+
                 '0,1,testData,2\n'+
                  '---\n'+
                 'default\n'+
@@ -202,7 +203,7 @@ Database meta::relational::tests::dbSpecificTests::sqlQueryTests::namingTests::m
     (
       ABCDEFGHIJKLMNOPQRSTUVWXYZ INT PRIMARY KEY,
       "0123456789" INT,
-      "!\"#$%&()*+-./:;<=>?@[\]^_`{|}" VARCHAR(20),   //single and double quotes not working
+      "!\\"#$%&()*+-./:;<=>?@[\]^_`{|}" VARCHAR(20),   //single and double quotes not working
       abcdefghijklmnopqrstuvwxzy INT
     )
 
@@ -249,7 +250,7 @@ Mapping meta::relational::tests::dbSpecificTests::sqlQueryTests::namingTests::te
        capitalAlphabets: [myDB] testTable3.ABCDEFGHIJKLMNOPQRSTUVWXYZ,
        smallAlphabets: [myDB] testTable3.abcdefghijklmnopqrstuvwxzy,
        numbers: [myDB] testTable3."0123456789",
-       specialCharacters: [myDB] testTable3."!\"#$%&()*+-./:;<=>?@[\]^_`{|}"
+       specialCharacters: [myDB] testTable3."!\\"#$%&()*+-./:;<=>?@[\]^_`{|}"
     }
 
     MyClassD: Relational