Merge pull request #152 from smallAreaHealthStatisticsUnit/bug/#79_SQ…

…L_Server_bulk_insert_needs_unicode

Support Unicode in SQL Server BULK INSERT statements.

rifDataLoaderTool/src/main/java/org/sahsu/rif/dataloader/datastorage/ms/MSSQLExtractWorkflowManager.java

@@ -264,7 +264,9 @@ private void importCSVFile(
 			queryFormatter.addQueryPhrase(destinationTableName);
 			queryFormatter.addQueryPhrase(" FROM '");
 			queryFormatter.addQueryPhrase(csvFilePath);
-			queryFormatter.addQueryPhrase("' WITH ( FIRSTROW = 2, FIELDTERMINATOR = ',', ROWTERMINATOR = '\\n', TABLOCK)");
+			queryFormatter.addQueryPhrase("' WITH ( FIRSTROW = 2, FIELDTERMINATOR = ',', "
+			                              + "ROWTERMINATOR = '\\n', TABLOCK, "
+			                              + "DATAFILETYPE = 'widechar')");
 			// by specifying FIRSTROW = 2 we skip the header row
 
 			System.out.println(queryFormatter.generateQuery());

rifDataLoaderTool/src/main/java/org/sahsu/rif/dataloader/scriptgenerator/ms/MSBulkInsertQueryFormatter.java

@@ -123,7 +123,8 @@ public String generateQuery() {
 		addQueryPhrase("',");
 		finishLine();
 		addQueryLine(1, "TABLOCK,");
-		addQueryLine(1, "FIRSTROW=2");
+		addQueryLine(1, "FIRSTROW=2,");
+		addQueryLine(1, "DATAFILETYPE = 'widechar'");
 		addQueryPhrase(0, ")");
 
 		return super.generateQuery();

rifDatabase/DataLoaderData/SAHSULAND/ms_run_data_loader.sql

@@ -117,7 +117,8 @@ WITH
 (
    FIRSTROW=2,
    FORMATFILE = '$(pwd)/pop_sahsuland_pop.fmt',
-   TABLOCK
+   TABLOCK,
+   DATAFILETYPE = 'widechar'
 );
 GO
 ALTER TABLE rif_data.POP_SAHSULAND_POP ADD CONSTRAINT pop_sahsuland_pop_pk PRIMARY KEY CLUSTERED(YEAR,AGE_SEX_GROUP,SAHSU_GRD_LEVEL4);
@@ -275,7 +276,8 @@ WITH
 (
    FORMATFILE = '$(pwd)/num_sahsuland_cancer.fmt',
    TABLOCK,
-   FIRSTROW=2
+   FIRSTROW=2,
+   DATAFILETYPE = 'widechar'
 );
 GO
 ALTER TABLE rif_data.NUM_SAHSULAND_CANCER ADD CONSTRAINT num_sahsuland_cancer_pk PRIMARY KEY CLUSTERED(YEAR,AGE_SEX_GROUP,SAHSU_GRD_LEVEL4,ICD);
@@ -469,7 +471,8 @@ WITH
 (
    FORMATFILE = '$(pwd)/covar_sahsuland_covariates3.fmt',
    TABLOCK,
-   FIRSTROW=2
+   FIRSTROW=2,
+   DATAFILETYPE = 'widechar'
 );
 GO
 ALTER TABLE rif_data.COVAR_SAHSULAND_COVARIATES3 ADD CONSTRAINT covar_sahsuland_covariates3_pk PRIMARY KEY CLUSTERED(YEAR,SAHSU_GRD_LEVEL3);
@@ -581,7 +584,8 @@ WITH
 (
    FORMATFILE = '$(pwd)/covar_sahsuland_covariates4.fmt',
    TABLOCK,
-   FIRSTROW=2
+   FIRSTROW=2,
+   DATAFILETYPE = 'widechar'
 );
 GO
 ALTER TABLE rif_data.COVAR_SAHSULAND_COVARIATES4 ADD CONSTRAINT covar_sahsuland_covariates4_pk PRIMARY KEY CLUSTERED(YEAR,SAHSU_GRD_LEVEL4);

rifDatabase/DataLoaderData/SAHSULandUpdate/rif_mssql_new_sahsuland.sql

@@ -69,7 +69,8 @@ WITH
 (
 	FIRSTROW = 2,
 	FORMATFILE = '$(pwd)/covar_sahsuland_covariates3.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -141,7 +142,8 @@ WITH
 (
 	FIRSTROW = 2,
 	FORMATFILE = '$(pwd)/covar_sahsuland_covariates4.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -213,7 +215,8 @@ WITH
 (
 	FIRSTROW = 2,
 	FORMATFILE = '$(pwd)/num_sahsuland_cancer.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -299,7 +302,8 @@ WITH
 (
 	FIRSTROW = 2,
 	FORMATFILE = '$(pwd)/pop_sahsuland_pop.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -406,4 +410,4 @@ GO
 QUIT
 
 --
--- Eof
+-- Eof

rifDatabase/GeospatialData/tileMaker/mssql_SAHSULAND.sql

@@ -790,7 +790,8 @@ FROM '$(pwd)/sahsu_grd_level1.csv'	-- Note use of pwd; set via -v pwd="%cd%" in
 WITH
 (
 	FORMATFILE = '$(pwd)/mssql_sahsu_grd_level1.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -2115,7 +2116,8 @@ FROM '$(pwd)/sahsu_grd_level2.csv'	-- Note use of pwd; set via -v pwd="%cd%" in
 WITH
 (
 	FORMATFILE = '$(pwd)/mssql_sahsu_grd_level2.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -3395,7 +3397,8 @@ FROM '$(pwd)/sahsu_grd_level3.csv'	-- Note use of pwd; set via -v pwd="%cd%" in
 WITH
 (
 	FORMATFILE = '$(pwd)/mssql_sahsu_grd_level3.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar'
 );
 GO
 
@@ -4765,7 +4768,8 @@ FROM '$(pwd)/sahsu_grd_level4.csv'	-- Note use of pwd; set via -v pwd="%cd%" in
 WITH
 (
 	FORMATFILE = '$(pwd)/mssql_sahsu_grd_level4.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 

rifDatabase/GeospatialData/tileMaker/rif_mssql_SAHSULAND.sql

@@ -1105,7 +1105,8 @@ FROM '$(pwd)\mssql_lookup_sahsu_grd_level1.csv'	-- Note use of pwd; set via -v p
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_lookup_sahsu_grd_level1.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -1115,7 +1116,8 @@ FROM '$(pwd)\mssql_lookup_sahsu_grd_level2.csv'	-- Note use of pwd; set via -v p
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_lookup_sahsu_grd_level2.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -1125,7 +1127,8 @@ FROM '$(pwd)\mssql_lookup_sahsu_grd_level3.csv'	-- Note use of pwd; set via -v p
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_lookup_sahsu_grd_level3.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -1135,7 +1138,8 @@ FROM '$(pwd)\mssql_lookup_sahsu_grd_level4.csv'	-- Note use of pwd; set via -v p
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_lookup_sahsu_grd_level4.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -1394,7 +1398,8 @@ FROM '$(pwd)\mssql_hierarchy_sahsuland.csv'	-- Note use of pwd; set via -v pwd="
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_hierarchy_sahsuland.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -1795,7 +1800,8 @@ FROM '$(pwd)\mssql_geometry_sahsuland.csv'	-- Note use of pwd; set via -v pwd="%
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_geometry_sahsuland.fmt',	-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -2206,7 +2212,8 @@ FROM '$(pwd)\mssql_adjacency_sahsuland.csv'	-- Note use of pwd; set via -v pwd="
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_adjacency_sahsuland.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -4825,7 +4832,8 @@ FROM '$(pwd)\mssql_t_tiles_sahsu_grd_level1.csv'	-- Note use of pwd; set via -v
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_t_tiles_sahsu_grd_level1.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -4846,7 +4854,8 @@ FROM '$(pwd)\mssql_t_tiles_sahsu_grd_level2.csv'	-- Note use of pwd; set via -v
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_t_tiles_sahsu_grd_level2.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -4867,7 +4876,8 @@ FROM '$(pwd)\mssql_t_tiles_sahsu_grd_level3.csv'	-- Note use of pwd; set via -v
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_t_tiles_sahsu_grd_level3.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 
@@ -4888,7 +4898,8 @@ FROM '$(pwd)\mssql_t_tiles_sahsu_grd_level4.csv'	-- Note use of pwd; set via -v
 WITH
 (
 	FORMATFILE = '$(pwd)\mssql_t_tiles_sahsu_grd_level4.fmt',		-- Use a format file
-	TABLOCK					-- Table lock
+	TABLOCK,					-- Table lock
+    DATAFILETYPE = 'widechar' -- Handle Unicode
 );
 GO
 

rifDatabase/SQLserver/installation/rif40_import_sahsuland.sql

@@ -82,7 +82,8 @@ FROM '$(path)\SQLserver\sahsuland_dev\rif_data\rif40_columns.txt'
 WITH
 (
 FIELDTERMINATOR = '|',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 BULK
@@ -91,7 +92,8 @@ FROM '$(path)\SQLserver\sahsuland_dev\rif_data\rif40_tables_and_views.txt'
 WITH
 (
 FIELDTERMINATOR = '|',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 
@@ -172,7 +174,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_version.csv'
 WITH
 (
 FIELDTERMINATOR = ',',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 
@@ -182,7 +185,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_age_groups.csv'
 WITH
 (
 FIELDTERMINATOR = ',',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 BULK
@@ -191,7 +195,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_age_group_names.csv'
 WITH
 (
 FIELDTERMINATOR = ',',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 
@@ -201,7 +206,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_reference_tables.csv'
 WITH
 (
 FIELDTERMINATOR = ',',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 
@@ -211,7 +217,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_outcomes.csv'
 WITH
 (
 FIELDTERMINATOR = ',',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 
@@ -221,7 +228,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_health_study_themes.csv'
 WITH
 (
 FIELDTERMINATOR = ',',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 /*
@@ -273,7 +281,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_outcome_groups.csv'
 WITH
 (
 FIELDTERMINATOR = ',',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 
@@ -283,7 +292,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_table_outcomes.csv'
 WITH
 (
 FIELDTERMINATOR = ',',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 
@@ -307,7 +317,8 @@ FROM '$(path)\Postgres\sahsuland\data\t_rif40_parameters.csv'
 WITH
 (
 FIELDTERMINATOR = ',',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 
 --
@@ -321,7 +332,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_triggers_mssql.csv'
 WITH
 (
 FIELDTERMINATOR = '|',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 
@@ -331,7 +343,8 @@ FROM '$(path)\Postgres\sahsuland\data\rif40_predefined_groups_mssql.csv'
 WITH
 (
 FIELDTERMINATOR = '|',
-ROWTERMINATOR = '0x0a'
+ROWTERMINATOR = '0x0a',
+DATAFILETYPE = 'widechar'
 )
 GO
 /*

rifNodeServices/dataLoader/tile-maker_fileParser.js

@@ -278,7 +278,7 @@ function shpConvertInput(files, shpConvertInputCallback) {
 									data=fileContainedInZipFile.asArrayBuffer();
 								}
 								else {
-									console.log("asyncSeriesIteree() WARNING! Ignoring[" + ZipIndex + "]: " + zipName + "; size: " + 
+									console.log("asyncSeriesIteree() WARNING! Ignoring extension: " + zipExt + "; index: " + ZipIndex + ": " + zipName + "; size: " + 
 										fileContainedInZipFile._data.length || fileContainedInZipFile._data.uncompressedSize + " bytes");
 		//							zipMsg+="<br>Zip file[" + noZipFiles + "]: file: " + zipName + 
 		//								"; expanded: " + unzipPct +