Use d3-dsv package for parsing tsv/csv files.

config-examples.yaml

@@ -197,6 +197,39 @@ dataCubes:
         title: Records count
         formula: $main.count()
 
+  - name: unemployment
+    title: Unemployment Across Industries
+    description: |
+      Data cube with unemployment count across industriess in 2000-2010.
+      ---
+      *Based on vega-datasets*. [Source repository](https://github.com/vega/vega-datasets/blob/master/data/unemployment-across-industries.json)
+    clusterName: native
+    source: assets/data/unemployment-2016-02-18.csv
+    timeAttribute: date
+
+    refreshRule:
+      rule: fixed
+      time: 2010-02-01T08:00:00.000Z
+
+    defaultDuration: P1M
+    defaultSortMeasure: count
+    defaultSelectedMeasures: [ "count" ]
+    introspection: no-autofill
+    dimensions:
+      - name: date
+        title: Date
+        kind: time
+        formula: $date
+
+      - name: industry
+        title: Industry
+        formula: $industry
+
+    measures:
+      - name: count
+        title: Count
+        formula: $main.sum($count)
+
   - name: wiki
     title: Wikipedia
     description: |

cypress/integration/csv-data-source.spec.js

@@ -0,0 +1,71 @@
+context('CSV Data Source', () => {
+
+  const count = () => cy.get('.total .measure-value');
+  const leftGutter = () => cy.get('.table .left-gutter');
+  const body = () => cy.get('.table .body');
+  const splitValue = (idx) => leftGutter().get(`.split-value:nth-child(${idx})`);
+  const measureValue = (idx) => body().get(`.measure-row:nth-child(${idx}) .measure-label`);
+
+  function assertDates(...dates) {
+    dates.forEach((date, idx) => {
+      splitValue(idx + 1).should('contain', date);
+    });
+  }
+
+  function assertValues(...values) {
+    values.forEach((value, idx) => {
+      measureValue(idx + 1).should('contain', value);
+    });
+  }
+
+  describe('Data load', () => {
+    const urls = {
+      baseView: 'http://localhost:9090/#unemployment/4/N4IgbglgzgrghgGwgLzgFwgewHYgFwhqZqJQgA0408SqGOAygKZobYDmZe2MCClGALZNkOJvhABRNAGMA9AFUAKgGEKIAGYQEaJgCcuAbVBoAngAdxBIeMp6mGiQBN0twhGEAFfVidHQTjB66Fi4BJ4AjACy6lC65vgAtBECFlYgCK5xIAC+ALr5lFDmSGhGeUU+TP4g9hr6TNgy6TKYMNho6hqYeoLo+CZpzg5wvJ2UYIgw6bmplhLCcLD2uRUg5hDY2ExOACIejVCh5ZQbWzsMPZ0Ere2dOUA=',
+      lastYearTable: 'http://localhost:9090/#unemployment/4/N4IgbglgzgrghgGwgLzgFwgewHYgFwhpwBGCApiADTjTxKoY4DKZaG2A5lPqAMaYIEcAA5QyAJUwB3bngBmiMQF9qGALZlkOCgQCiaXgHoAqgBUAwlRByICNGQBOsgNqg0AT2E7CEDVYdkcvggACboFKq+ZAAKjlghLqAhMA7oWLgE0QCMAJpWUPbC+AC0Waqe3kL2BSBKALr11FDCSGiJhBXB6hGhUdhQ6cFh9lYcqdgwQg4QHsHZALL5mA5oPCABQQTDPR5eQ30DOFYhEAG8jBkgcFC8ZNgnnLXUSGoz+BOCjSBi02TtG447rdgvwYNhVtQ5Ms1Og1rtvCFAnBJhDwIgYN4nh09gQNNcUhQvsIINhsGQQgARA7pFx1ajE0nkpjLVYEUHg2pAA='
+    };
+
+    it('should load last month count', () => {
+      cy.visit(urls.baseView);
+
+      count().should('contain', '15.0 k');
+    });
+
+    it('should load table for last year counts', () => {
+      cy.visit(urls.lastYearTable);
+
+      assertDates(
+        'Total',
+        '1 Mar 2009',
+        '1 Apr 2009',
+        '1 May 2009',
+        '1 Jun 2009',
+        '1 Jul 2009',
+        '1 Aug 2009',
+        '1 Sep 2009',
+        '1 Oct 2009',
+        '1 Nov 2009',
+        '1 Dec 2009',
+        '1 Jan 2010',
+        '1 Feb 2010'
+      );
+
+      assertValues(
+        '163.7 k',
+        '13.2 k',
+        '12.5 k',
+        '13.0 k',
+        '13.7 k',
+        '13.8 k',
+        '13.6 k',
+        '13.4 k',
+        '13.5 k',
+        '13.2 k',
+        '13.7 k',
+        '15.1 k',
+        '15.0 k'
+      );
+
+    });
+
+  });
+});

cypress/integration/home-page.spec.js

@@ -7,11 +7,14 @@ context("Home Page", () => {
     cy.visit("http://localhost:9090");
   });
 
-
   it("should load Covid datacube", () => {
     getCubeCard("COVID").should("exist");
   });
 
+  it("should load Unemployment datacube", () => {
+    getCubeCard("Unemployment").should("exist");
+  });
+
   it("should load Wikipedia datacube", () => {
     getCubeCard("Wikipedia").should("exist");
   });

package.json

@@ -105,6 +105,7 @@
     "@types/chai-datetime": "0.0.33",
     "@types/compression": "0.0.35",
     "@types/d3": "3.5.40",
+    "@types/d3-dsv": "2.0.0",
     "@types/d3-scale": "2.1.1",
     "@types/enzyme": "3.1.11",
     "@types/express": "4.11.0",
@@ -144,6 +145,7 @@
     "css-loader": "2.1.1",
     "css-modules-require-hook": "4.2.3",
     "cypress": "3.4.1",
+    "d3-dsv": "2.0.0",
     "enzyme": "3.3.0",
     "enzyme-adapter-react-16": "1.7.1",
     "ignore-styles": "5.0.1",

src/server/utils/file-manager/file-manager.ts

@@ -20,7 +20,7 @@ import * as path from "path";
 import { Dataset, Expression, PseudoDatum } from "plywood";
 import { Logger } from "../../../common/logger/logger";
 import { noop } from "../../../common/utils/functional/functional";
-import { parseData } from "../../../common/utils/parser/parser";
+import { parseData } from "../parser/parser";
 
 export function getFileData(filePath: string): Promise<any[]> {
   return fs.readFile(filePath, "utf-8")

src/common/utils/parser/parser.ts → src/server/utils/parser/parser.ts

@@ -15,14 +15,14 @@
  * limitations under the License.
  */
 
-import * as d3 from "d3";
+import { autoType, csvParse, tsvParse } from "d3-dsv";
 
 export function parseCSV(text: string): any[] {
-  return d3.csv.parse(text);
+  return csvParse(text, autoType);
 }
 
 export function parseTSV(text: string): any[] {
-  return d3.tsv.parse(text);
+  return tsvParse(text, autoType);
 }
 
 export function parseJSON(text: string): any[] {