Revert "support comma in column value for service test csv data (#1899)…

…" (#1978)
finos · Jun 29, 2023 · 90d896b · 90d896b
1 parent ac8df8b
commit 90d896b
Show file tree

Hide file tree

Showing 6 changed files with 8 additions and 156 deletions.
diff --git a/...-service/src/test/java/org/finos/legend/engine/testable/service/TestServiceTestSuite.java b/...-service/src/test/java/org/finos/legend/engine/testable/service/TestServiceTestSuite.java
@@ -3081,22 +3081,6 @@ public void testPassingRelationalWithParams()
         Assert.assertEquals(passed.testSuiteId, "testSuite1");
     }
 
-    @Test
-    public void testRelationalServiceWithCommmaInCSV()
-    {
-        // setup
-        List<TestResult> relationalTestResult = executeServiceTest("testable/relational/", "legend-testable-relational-model.pure", "legend-testable-relational-service-csv-test-data.pure","service::RelationalServiceWithCSV");
-        // Assertions
-        Assert.assertEquals(relationalTestResult.size(), 1);
-        TestResult testResult = relationalTestResult.get(0);
-        Assert.assertEquals(testResult.testable, "service::RelationalServiceWithCSV");
-        Assert.assertTrue(testResult instanceof TestExecuted);
-        Assert.assertEquals(TestExecutionStatus.PASS, ((TestExecuted) testResult).testExecutionStatus);
-        TestExecuted passed = (TestExecuted) testResult;
-        Assert.assertEquals(passed.atomicTestId, "test_1");
-        Assert.assertEquals(passed.testSuiteId, "testSuite_1");
-    }
-
     @Test
     public void testPassingRelationalWithEnumParams()
     {

diff --git a/.../test/resources/testable/relational/legend-testable-relational-service-csv-test-data.pure b/.../test/resources/testable/relational/legend-testable-relational-service-csv-test-data.pure
diff --git a/...c/test/resources/testable/relational/legend-testable-relational-service-embeddedData.pure b/...c/test/resources/testable/relational/legend-testable-relational-service-embeddedData.pure
@@ -23,7 +23,7 @@ Service service::SimpleRelationalPassWithSpecialEmbeddedData
           #{
             default.PersonTable:
               'id,firm_id,firstName,lastName\n' +
-              '1,1,"John;\'",Doe\n' +
+              '1,1,John;\'",Doe\n' +
               '2,1,Nicole,Smith\n' +
               '3,2,Time,Smith\n';
 
@@ -48,7 +48,7 @@ Service service::SimpleRelationalPassWithSpecialEmbeddedData
                   ExternalFormat
                   #{
                     contentType: 'application/json';
-                    data: '[{"Employees/First Name":"\\"John;\'\\"","Employees/Last Name":"Doe","Legal Name":"Finos"},{"Employees/First Name":"Nicole","Employees/Last Name":"Smith","Legal Name":"Finos"},{"Employees/First Name":"Time","Employees/Last Name":"Smith","Legal Name":"Apple"}]\n';
+                    data: '[{"Employees/First Name":"John;\'\\"","Employees/Last Name":"Doe","Legal Name":"Finos"},{"Employees/First Name":"Nicole","Employees/Last Name":"Smith","Legal Name":"Finos"},{"Employees/First Name":"Time","Employees/Last Name":"Smith","Legal Name":"Apple"}]\n';
                   }#;
               }#
           ]

diff --git a/...e-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure b/...e-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure
@@ -115,46 +115,10 @@ function <<access.private>> meta::alloy::service::execution::splitWithEmptyValue
    let delimForEmptyCsvField = '|EMPTY|';
 
    $s->meta::alloy::service::execution::replaceWithEmptyValue($delimForEmptyCsvField)
-        ->meta::alloy::service::execution::splitCsvRow(',')
+        ->split(',')
         ->map(csv_val| if($csv_val == $delimForEmptyCsvField, | '' ,| $csv_val));
 }
 
-Class <<access.private>> meta::alloy::service::execution::SplitterState
-{
-   tokens:String[*];
-   inQuotes:Boolean[1];
-   isPrevEscapeChar: Boolean[1];
-   currentTokenChars:String[*];
-}
-
-function meta::alloy::service::execution::splitCsvRow(s: String[1], delimiter: String[1]): String[*]
-{
-  let charArray = chunk($s, 1);
-  let finalSplitterState = $charArray->fold({currentChar,splitterState |
-                  if ($currentChar == $delimiter,
-                    | if(!$splitterState.inQuotes,
-                      | ^$splitterState(tokens = $splitterState.tokens->add($splitterState.currentTokenChars->makeString()->trim()), inQuotes= false, isPrevEscapeChar = false, currentTokenChars=[]),
-                      | ^$splitterState(tokens = $splitterState.tokens, isPrevEscapeChar = false, currentTokenChars = $splitterState.currentTokenChars->add($currentChar))
-                      ),
-                    | if ($currentChar == '"',
-                        | if ($splitterState.isPrevEscapeChar,
-                        | ^$splitterState(isPrevEscapeChar = false, currentTokenChars = $splitterState.currentTokenChars->add($currentChar));,
-                        | if(!$splitterState.inQuotes,
-                            | ^$splitterState(inQuotes = true, currentTokenChars = $splitterState.currentTokenChars->add($currentChar)),
-                            | ^$splitterState(inQuotes = false, currentTokenChars = $splitterState.currentTokenChars->add($currentChar))
-                            )
-                           ),
-                        | if ( $currentChar == '\\',
-                           | ^$splitterState(isPrevEscapeChar = true, currentTokenChars = $splitterState.currentTokenChars->add($currentChar)),
-                           | ^$splitterState(isPrevEscapeChar = false, currentTokenChars = $splitterState.currentTokenChars->add($currentChar))
-                             )
-                       )
-                    )                            
-                  }, ^meta::alloy::service::execution::SplitterState(tokens=[], inQuotes= false, isPrevEscapeChar = false, currentTokenChars=[]));
-
-  if($finalSplitterState.currentTokenChars != [],|$finalSplitterState.tokens->add($finalSplitterState.currentTokenChars->makeString()->trim()), | $finalSplitterState.tokens);
-}
-
 function <<access.private>> meta::alloy::service::execution::replaceWithEmptyValue(s:String[1], delimForEmptyCsvField: String[1]) : String[1]
 {
    let news = if($s->endsWith(','), | $s + $delimForEmptyCsvField, |$s);

diff --git a/...tore-pure/src/main/resources/core_relational/relational/sqlQueryToString/DDL/testDDL.pure b/...tore-pure/src/main/resources/core_relational/relational/sqlQueryToString/DDL/testDDL.pure
@@ -112,45 +112,4 @@ function <<test.Test>>  meta::relational::tests::ddl::testSetupDataSqlGeneration
    'insert into personTable (ID,FIRSTNAME,LASTNAME,AGE,ADDRESSID,FIRMID,MANAGERID) values (1,\'Peter\',\'Smith\',23,1,1,2);'
    ];
   assertSameElements($sqls,$expectedSqls);
-}  
-
-function <<test.Test>>  meta::relational::tests::ddl::testSetupDataSqlGenerationWithColumnValueHasDelimiterAndQuotes():Boolean[1]
-{
-  let records ='default\n'+
-                'personTable\n'+
-                'id, firstName, lastName, age, addressId, firmId, managerId\n'+
-                '1, Peter,  "I\'m Smith, Jr",23,1,1,2';
-
-  let sqls = meta::alloy::service::execution::setUpDataSQLs($records, meta::relational::tests::dbInc, meta::relational::functions::sqlQueryToString::createDbConfig(DatabaseType.H2));
-
-  let expectedSqls= [
-   'Drop schema if exists productSchema cascade;',
-   'Create Schema productSchema;',
-   'Drop schema if exists default cascade;',
-   'Create Schema default;',
-   'Drop table if exists productSchema.productTable;',
-   'Create Table productSchema.productTable(ID INT NOT NULL,NAME VARCHAR(200) NULL, PRIMARY KEY(ID));',
-   'Drop table if exists personTable;',
-   'Create Table personTable(ID INT NOT NULL,FIRSTNAME VARCHAR(200) NULL,LASTNAME VARCHAR(200) NULL,AGE INT NULL,ADDRESSID INT NULL,FIRMID INT NULL,MANAGERID INT NULL, PRIMARY KEY(ID));',
-   'Drop table if exists PersonTableExtension;',
-   'Create Table PersonTableExtension(ID INT NOT NULL,FIRSTNAME VARCHAR(200) NULL,LASTNAME VARCHAR(200) NULL,AGE INT NULL,ADDRESSID INT NULL,FIRMID INT NULL,MANAGERID INT NULL,birthDate DATE NULL, PRIMARY KEY(ID));',
-   'Drop table if exists differentPersonTable;',
-   'Create Table differentPersonTable(ID INT NOT NULL,FIRSTNAME VARCHAR(200) NULL,LASTNAME VARCHAR(200) NULL,AGE INT NULL,ADDRESSID INT NULL,FIRMID INT NULL,MANAGERID INT NULL, PRIMARY KEY(ID));',
-   'Drop table if exists firmTable;',
-   'Create Table firmTable(ID INT NOT NULL,LEGALNAME VARCHAR(200) NULL,ADDRESSID INT NULL,CEOID INT NULL, PRIMARY KEY(ID));',
-   'Drop table if exists firmExtensionTable;',
-   'Create Table firmExtensionTable(firmId INT NOT NULL,legalName VARCHAR(200) NULL,establishedDate DATE NULL, PRIMARY KEY(firmId));',
-   'Drop table if exists otherFirmTable;',
-   'Create Table otherFirmTable(ID INT NOT NULL,LEGALNAME VARCHAR(200) NULL,ADDRESSID INT NULL, PRIMARY KEY(ID));',
-   'Drop table if exists addressTable;',
-   'Create Table addressTable(ID INT NOT NULL,TYPE INT NULL,NAME VARCHAR(200) NULL,STREET VARCHAR(100) NULL,COMMENTS VARCHAR(100) NULL, PRIMARY KEY(ID));',
-   'Drop table if exists locationTable;',
-   'Create Table locationTable(ID INT NOT NULL,PERSONID INT NULL,PLACE VARCHAR(200) NULL,date DATE NULL, PRIMARY KEY(ID));',
-   'Drop table if exists placeOfInterestTable;',
-   'Create Table placeOfInterestTable(ID INT NOT NULL,locationID INT NOT NULL,NAME VARCHAR(200) NULL, PRIMARY KEY(ID,locationID));',
-   'Drop table if exists validPersonTable;',
-   'Create Table validPersonTable(ID INT NOT NULL,FIRSTNAME VARCHAR(200) NULL,LASTNAME VARCHAR(200) NULL,AGE INT NULL,ADDRESSID INT NULL,FIRMID INT NULL,MANAGERID INT NULL, PRIMARY KEY(ID));',
-   'insert into personTable (ID,FIRSTNAME,LASTNAME,AGE,ADDRESSID,FIRMID,MANAGERID) values (1,\'Peter\',\'"I\'\'m Smith, Jr"\',23,1,1,2);'
-   ];
-  assertSameElements($sqls,$expectedSqls);
-}
+}  
diff --git a/...c/main/resources/core_relational/relational/sqlQueryToString/testSuite/naming/naming.pure b/...c/main/resources/core_relational/relational/sqlQueryToString/testSuite/naming/naming.pure
@@ -116,8 +116,7 @@ function <<paramTest.BeforePackage>> meta::relational::tests::dbSpecificTests::s
   //  'insert into "test Table2"("Column&ID", "My#Data") values (3,\'Description A\');'
   //  ];
 
-   let records =
-               'default\n'+
+   let records ='default\n'+
                 'testTable\n'+
                 'ID, "Column Name with Space", "Column&A", "Column.ID"\n'+
                 '1,2,0,3\n'+
@@ -130,7 +129,7 @@ function <<paramTest.BeforePackage>> meta::relational::tests::dbSpecificTests::s
                  '---\n'+
                 'default\n'+
                 'testTable3\n'+
-                'ABCDEFGHIJKLMNOPQRSTUVWXYZ,"0123456789","!\\"#$%&()*+-./:;<=>?@[\]^_`{|}",abcdefghijklmnopqrstuvwxzy\n'+
+                'ABCDEFGHIJKLMNOPQRSTUVWXYZ,"0123456789","!\"#$%&()*+-./:;<=>?@[\]^_`{|}",abcdefghijklmnopqrstuvwxzy\n'+
                 '0,1,testData,2\n'+
                  '---\n'+
                 'default\n'+
@@ -203,7 +202,7 @@ Database meta::relational::tests::dbSpecificTests::sqlQueryTests::namingTests::m
     (
       ABCDEFGHIJKLMNOPQRSTUVWXYZ INT PRIMARY KEY,
       "0123456789" INT,
-      "!\\"#$%&()*+-./:;<=>?@[\]^_`{|}" VARCHAR(20),   //single and double quotes not working
+      "!\"#$%&()*+-./:;<=>?@[\]^_`{|}" VARCHAR(20),   //single and double quotes not working
       abcdefghijklmnopqrstuvwxzy INT
     )
 
@@ -250,7 +249,7 @@ Mapping meta::relational::tests::dbSpecificTests::sqlQueryTests::namingTests::te
        capitalAlphabets: [myDB] testTable3.ABCDEFGHIJKLMNOPQRSTUVWXYZ,
        smallAlphabets: [myDB] testTable3.abcdefghijklmnopqrstuvwxzy,
        numbers: [myDB] testTable3."0123456789",
-       specialCharacters: [myDB] testTable3."!\\"#$%&()*+-./:;<=>?@[\]^_`{|}"
+       specialCharacters: [myDB] testTable3."!\"#$%&()*+-./:;<=>?@[\]^_`{|}"
     }
 
     MyClassD: Relational