feature/query endpoint backend (#1022)

* endpoint validation and helpers * timezone is optional and we need to support old parameter dataCubeName * pass whole settings and avoid functional-object pattern mismatch * tests for query endpoint * organize utils * split running query and obtaining query decorator (with some partial application shenaningans) * tests for mkurl for invalid inputs * viewDefinition2 fixtures for mkurl tests * timekeeper fixture for wiki data cube * normalize view definition mocks across wiki data cube and syntetic cube * use fixtures in query tests * better wording * tests for handle-request-errors * endpoints test only endpoint logic, controllers are tested e2e
allegro · Feb 8, 2023 · 07e18c2 · 07e18c2
1 parent 9808c46
commit 07e18c2
Show file tree

Hide file tree

Showing 31 changed files with 873 additions and 315 deletions.
diff --git a/cypress/e2e/mkurl.cy.ts b/cypress/e2e/mkurl.cy.ts
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2017-2022 Allegro.pl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  ViewDefinitionConverter2Fixtures
+} from "../../src/common/view-definitions/version-2/view-definition-converter-2.fixtures";
+import { total } from "../../src/common/view-definitions/version-4/view-definition-4.fixture";
+
+interface MkurlResponse {
+  hash: string;
+}
+
+context("mkurl", () => {
+  it("should return hash for version 4 view definition", () => {
+    const body = {
+      dataCubeName: "wiki",
+      viewDefinitionVersion: "4",
+      viewDefinition: total
+    };
+
+    cy.request("POST", "http://localhost:9090/mkurl", body).then((hash: Cypress.Response<MkurlResponse>) => {
+      expect(hash.status).to.eq(200);
+      expect(hash.body.hash).to.eq("#wiki/4/N4IgbglgzgrghgGwgLzgFwgewHYgFwhqZqJQgA0408SqGOAygKZobYDmZe2MCClGALZNkOJvhABRNAGMA9AFUAKgGEKIAGYQEaJgCcuAbVBoAngAdxBIeMp6mGiTfU2ACvqwATI6E8w96Fi4BK4AjAAi6lC65vgAtKECFlaaCJiY9p4gAL4AunmUUOZIaEa5hR5MPiD2GvpM2DIpMpgw2GjqGhmC6PgmyRKeDnC8HZRgiDApOUmWEsJwsPYzoLX1jSlwnkNZlF16PR14/XMEQxojOuoTCFMSM4QDBAtL4gUg5hDY2Eye4RDCbBQIJlSifb6/BgZI4gLY7HJAA===");
+    });
+  });
+
+  it("should return hash for version 2 view definition", () => {
+    const body = {
+      dataCubeName: "wiki",
+      viewDefinitionVersion: "2",
+      viewDefinition: ViewDefinitionConverter2Fixtures.fullTable()
+    };
+
+    cy.request("POST", "http://localhost:9090/mkurl", body).then((hash: Cypress.Response<MkurlResponse>) => {
+      expect(hash.status).to.eq(200);
+      expect(hash.body.hash).to.eq("#wiki/4/N4IgbglgzgrghgGwgLzgFwgewHYgFwhpwBGCApiADTjTxKoY4DKZaG2A5lPqAMaYIEcAA5QyAJUwB3bngBmiMQF9qGALZlkOCgQCiaXgHoAqgBUAwlRByICNGQBOsgNqg0AT2E7CEDVYdkcvg+fqq+EnCcZC6gUEQOaMEATAAMAIwArAC0KQCcWWlJpikpeCVlKQB0JSkAWlZk2AAmyenZeQUAzMWl5SXVJfVKALoqbp7ecQ4QnP6BwbwAFpHYZAhWcLyMuAQzVmCIMNH4ziCNIMPU2JiJ8opkY4QTwcSYAmSRc0G7UJKvidQDggji40A4jpcQNdbmCjo8PF5glMZhwvsFhHAOBRqJttgscEQZtxAYdjnhTgApRxkNTuC5XG74BQIZSUcaIgjI2bUALfEAwMQOczLJwbLZYHYgNToJb7UkuEAAPQAggASelQxl3FkPNlPDlQmBqYiONEEfhqDTYNAAGUaHDQiys0KZ9x5kSxMRAcTgCXwnWojRaeGwMEE1FeMGashAzgAlCARiNqFBhEg0F6EZMwSirE1wtgoBKFstsKt1inMH68KBecEmmsiFYs0jHBBjtR8wFxTh69FeEHc9QvNNMMGQInqEg1BBbhkUvDngRXu9Pp2C0Xez8/ozK9Xa/MCA27HBm0vvW2OyAu2Qe5KG1AB80hyAR1hx5OQNPZ/gMouDaGxqmuuVqbpKFpWra9qOlYHAOJEYa+rOdJ4GkKR7rcB58seTaqOegrtsS14QN2eJHv2g7cq+bZjsEn7fnO/7eOo2LEaBxYECxsHwaGQjTB4wQAAqmGkAASVhQFWmEgHW5EnmeBoEVeN53n2j6Uaiw40R+Khfr4P4hmGCDJhe0xkq4MmBNS2ADn28nUHIVbStJLbkQoYYAuApJ0bprlSh8sABJOWHWbZ5qYFGnmOQ4zk8Pq3gNu5dhysC3ifn5GhwIFFCPLyoXeHATQNi0DlOegcV+YlcAeSlRw+XhBqZdliaQsIMyrE0AAiG4SgqSwrGszpwBoqabKx0A7okrXtWQTRMFJdlNkoQA=");
+    });
+  });
+});
diff --git a/src/common/models/timekeeper/timekeeper.fixtures.ts b/src/common/models/timekeeper/timekeeper.fixtures.ts
@@ -28,4 +28,11 @@ export class TimekeeperFixtures {
   static fixed() {
     return Timekeeper.fromJS(TimekeeperFixtures.fixedJS());
   }
+
+  static wiki() {
+    return Timekeeper.fromJS({
+      timeTags: {},
+      nowOverride: new Date("2015-09-13T00:00:00.000Z")
+    });
+  }
 }
diff --git a/src/common/view-definitions/test/view-definition.fixture.ts b/src/common/view-definitions/test/view-definition.fixture.ts
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2017-2022 Allegro.pl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ViewDefinition4 } from "../version-4/view-definition-4";
+import { total } from "../version-4/view-definition-4.fixture";
+import { measures } from "./measure";
+
+const defaultSeries = measures.slice(0, 2).map(({ name }) => ({ reference: name }));
+
+export function mockViewDefinition(opts: Partial<ViewDefinition4> = {}): ViewDefinition4 {
+  return {
+    ...total,
+    series: defaultSeries,
+    ...opts
+  };
+}
diff --git a/src/common/view-definitions/version-2/view-definition-converter-2.fixtures.ts b/src/common/view-definitions/version-2/view-definition-converter-2.fixtures.ts
@@ -15,9 +15,10 @@
  * limitations under the License.
  */
 
+import { $ } from "plywood";
 import { ViewDefinition2 } from "./view-definition-2";
 
-const baseViewDefinition: ViewDefinition2 = {
+const baseTableViewDefinition: ViewDefinition2 = {
   visualization: "table",
   timezone: "Etc/UTC",
   filter: {
@@ -45,9 +46,123 @@ const baseViewDefinition: ViewDefinition2 = {
 };
 
 export class ViewDefinitionConverter2Fixtures {
-  static withFilterExpression(expression: any): ViewDefinition2 {
+  static totals(): ViewDefinition2 {
     return {
-      ...baseViewDefinition,
+      visualization: "totals",
+      timezone: "Etc/UTC",
+      filter: $("time").overlap(new Date("2015-09-12Z"), new Date("2015-09-13Z")),
+      pinnedDimensions: [],
+      singleMeasure: "count",
+      selectedMeasures: [],
+      splits: []   ,
+      multiMeasureMode: false
+    };
+  }
+
+  static fullTable(): ViewDefinition2 {
+    return {
+      visualization: "table",
+      timezone: "Etc/UTC",
+      filter:
+        $("time")
+          .overlap(new Date("2015-09-12Z"), new Date("2015-09-13Z"))
+          .and($("channel").overlap(["en"]))
+          .and($("isRobot").overlap([true]).not())
+          .and($("page").contains("Jeremy"))
+          .and($("userChars").match("^A$"))
+          .and($("commentLength").overlap([{ start: 3, end: null, type: "NUMBER_RANGE" }]))
+          .toJS(),
+      pinnedDimensions: ["channel", "namespace", "isRobot"],
+      pinnedSort: "delta",
+      singleMeasure: "delta",
+      selectedMeasures: ["delta", "count", "added"],
+      multiMeasureMode: true,
+      splits: [
+        {
+          expression: {
+            op: "ref",
+            name: "channel"
+          },
+          sortAction: {
+            op: "sort",
+            expression: {
+              op: "ref",
+              name: "delta"
+            },
+            direction: "descending"
+          },
+          limitAction: {
+            op: "limit",
+            value: 50
+          }
+        },
+        {
+          expression: {
+            op: "ref",
+            name: "isRobot"
+          },
+          sortAction: {
+            op: "sort",
+            expression: {
+              op: "ref",
+              name: "delta"
+            },
+            direction: "descending"
+          },
+          limitAction: {
+            op: "limit",
+            value: 5
+          }
+        },
+        {
+          expression: {
+            op: "ref",
+            name: "commentLength"
+          },
+          bucketAction: {
+            op: "numberBucket",
+            size: 10,
+            offset: 0
+          },
+          sortAction: {
+            op: "sort",
+            expression: {
+              op: "ref",
+              name: "delta"
+            },
+            direction: "descending"
+          },
+          limitAction: {
+            op: "limit",
+            value: 5
+          }
+        },
+        {
+          expression: {
+            op: "ref",
+            name: "time"
+          },
+          bucketAction: {
+            op: "timeBucket",
+            duration: "PT1H"
+          },
+          sortAction: {
+            op: "sort",
+            expression: {
+              op: "ref",
+              name: "delta"
+            },
+            direction: "descending"
+          }
+        }
+
+      ]
+    };
+  }
+
+  static tableWithFilterExpression(expression: any): ViewDefinition2 {
+    return {
+      ...baseTableViewDefinition,
       filter: {
         op: "overlap",
         operand: {
@@ -59,9 +174,9 @@ export class ViewDefinitionConverter2Fixtures {
     };
   }
 
-  static withFilterActions(actions: any[]): ViewDefinition2 {
+  static tableWithFilterActions(actions: any[]): ViewDefinition2 {
     return {
-      ...baseViewDefinition,
+      ...baseTableViewDefinition,
       filter: {
         op: "chain",
         expression: {
@@ -73,9 +188,9 @@ export class ViewDefinitionConverter2Fixtures {
     };
   }
 
-  static withSplits(splits: any[]): ViewDefinition2 {
+  static tableWithSplits(splits: any[]): ViewDefinition2 {
     return {
-      ...baseViewDefinition,
+      ...baseTableViewDefinition,
       splits
     };
   }

diff --git a/src/common/view-definitions/version-2/view-definition-converter-2.mocha.ts b/src/common/view-definitions/version-2/view-definition-converter-2.mocha.ts
@@ -64,7 +64,7 @@ describe("ViewDefinitionConverter2", () => {
     { label: "latest day", expression: latestDay, period: TimeFilterPeriod.LATEST }
   ].forEach(({ label, expression, period }) => {
     it(`converts ${label} bucket expression to time period`, () => {
-      const viewDefinition = ViewDefinitionConverter2Fixtures.withFilterExpression(expression);
+      const viewDefinition = ViewDefinitionConverter2Fixtures.tableWithFilterExpression(expression);
       const essence = new ViewDefinitionConverter2().fromViewDefinition(viewDefinition, clientAppSettings, wikiClientDataCube);
       const convertedClause = essence.filter.clauses.first();
 
@@ -74,7 +74,7 @@ describe("ViewDefinitionConverter2", () => {
   });
 
   it("converts filter with lookup expressions", () => {
-    const viewDefinition = ViewDefinitionConverter2Fixtures.withFilterActions([
+    const viewDefinition = ViewDefinitionConverter2Fixtures.tableWithFilterActions([
       {
         action: "in",
         expression: {
@@ -134,7 +134,7 @@ describe("ViewDefinitionConverter2", () => {
   });
 
   it("converts splits with lookup expressions", () => {
-    const viewDefinition = ViewDefinitionConverter2Fixtures.withSplits([{
+    const viewDefinition = ViewDefinitionConverter2Fixtures.tableWithSplits([{
       expression: {
         op: "chain",
         expression: {

diff --git a/src/common/view-definitions/version-4/test/filter-conversions.mocha.ts b/src/common/view-definitions/version-4/test/filter-conversions.mocha.ts
@@ -18,6 +18,7 @@ import { FilterClause, StringFilterAction, TimeFilterPeriod } from "../../../mod
 import { boolean, numberRange, stringWithAction, timePeriod, timeRange } from "../../../models/filter-clause/filter-clause.fixtures";
 import { Filter } from "../../../models/filter/filter";
 import { defaultTimeClause, mockEssence } from "../../test/essence.fixture";
+import { mockViewDefinition } from "../../test/view-definition.fixture";
 import { FilterClauseDefinition, filterDefinitionConverter } from "../filter-definition";
 import {
   booleanFilterDefinition,
@@ -28,7 +29,6 @@ import {
   stringFilterDefinition,
   timeRangeFilterDefinition
 } from "../filter-definition.fixtures";
-import { mockViewDefinition } from "../view-definition-4.fixture";
 import { assertConversionToEssence } from "./utils";
 
 const mockViewDefinitionWithFilters = (...filters: FilterClauseDefinition[]) =>

diff --git a/src/common/view-definitions/version-4/test/pinned-sort-conversions.mocha.ts b/src/common/view-definitions/version-4/test/pinned-sort-conversions.mocha.ts
@@ -17,8 +17,8 @@
 import { EMPTY_SERIES, SeriesList } from "../../../models/series-list/series-list";
 import { measureSeries, quantileSeries } from "../../../models/series/series.fixtures";
 import { mockEssence } from "../../test/essence.fixture";
+import { mockViewDefinition } from "../../test/view-definition.fixture";
 import { fromReference, quantileSeriesDefinition } from "../series-definition.fixtures";
-import { mockViewDefinition } from "../view-definition-4.fixture";
 import { assertConversionToEssence } from "./utils";
 
 describe("PinnedSort", () => {

diff --git a/src/common/view-definitions/version-4/test/series-conversions.mocha.ts b/src/common/view-definitions/version-4/test/series-conversions.mocha.ts
@@ -19,9 +19,9 @@ import { Series } from "../../../models/series/series";
 import { PERCENT_FORMAT } from "../../../models/series/series-format";
 import { measureSeries, quantileSeries } from "../../../models/series/series.fixtures";
 import { mockEssence } from "../../test/essence.fixture";
+import { mockViewDefinition } from "../../test/view-definition.fixture";
 import { SeriesDefinition } from "../series-definition";
 import { fromReference, measureSeriesDefinition, quantileSeriesDefinition } from "../series-definition.fixtures";
-import { mockViewDefinition } from "../view-definition-4.fixture";
 import { assertConversionToEssence } from "./utils";
 
 describe("Series", () => {

diff --git a/src/common/view-definitions/version-4/test/splits-conversions.mocha.ts b/src/common/view-definitions/version-4/test/splits-conversions.mocha.ts
@@ -23,9 +23,9 @@ import { Splits } from "../../../models/splits/splits";
 import { TABLE_MANIFEST } from "../../../visualization-manifests/table/table";
 import { assertEqlEssenceWithoutVisResolve } from "../../test/assertions";
 import { mockEssence } from "../../test/essence.fixture";
+import { mockViewDefinition } from "../../test/view-definition.fixture";
 import { SplitDefinition } from "../split-definition";
 import { numberSplitDefinition, stringSplitDefinition, timeSplitDefinition } from "../split-definition.fixtures";
-import { mockViewDefinition } from "../view-definition-4.fixture";
 import { assertConversionToEssence, toEssence } from "./utils";
 
 describe("Splits", () => {

diff --git a/src/common/view-definitions/version-4/test/time-shift-conversions.mocha.ts b/src/common/view-definitions/version-4/test/time-shift-conversions.mocha.ts
@@ -16,7 +16,7 @@
 
 import { TimeShift } from "../../../models/time-shift/time-shift";
 import { mockEssence } from "../../test/essence.fixture";
-import { mockViewDefinition } from "../view-definition-4.fixture";
+import { mockViewDefinition } from "../../test/view-definition.fixture";
 import { assertConversionToEssence } from "./utils";
 
 describe("TimeShift", () => {

diff --git a/src/common/view-definitions/version-4/test/timezone-conversions.mocha.ts b/src/common/view-definitions/version-4/test/timezone-conversions.mocha.ts
@@ -16,7 +16,7 @@
 
 import { Timezone } from "chronoshift";
 import { mockEssence } from "../../test/essence.fixture";
-import { mockViewDefinition } from "../view-definition-4.fixture";
+import { mockViewDefinition } from "../../test/view-definition.fixture";
 import { assertConversionToEssence } from "./utils";
 
 describe("Timezone", () => {

diff --git a/src/common/view-definitions/version-4/test/view-definition-converter-4.mocha.ts b/src/common/view-definitions/version-4/test/view-definition-converter-4.mocha.ts
@@ -16,7 +16,7 @@
 
 import { assertEqlEssence } from "../../test/assertions";
 import { mockEssence } from "../../test/essence.fixture";
-import { mockViewDefinition } from "../view-definition-4.fixture";
+import { mockViewDefinition } from "../../test/view-definition.fixture";
 import { ViewDefinitionConverter4 } from "../view-definition-converter-4";
 import { toEssence } from "./utils";
 

diff --git a/src/common/view-definitions/version-4/test/visualization-conversions.mocha.ts b/src/common/view-definitions/version-4/test/visualization-conversions.mocha.ts
@@ -19,7 +19,7 @@ import { ImmutableRecord } from "../../../utils/immutable-utils/immutable-utils"
 import { TABLE_MANIFEST } from "../../../visualization-manifests/table/table";
 import { TOTALS_MANIFEST } from "../../../visualization-manifests/totals/totals";
 import { mockEssence } from "../../test/essence.fixture";
-import { mockViewDefinition } from "../view-definition-4.fixture";
+import { mockViewDefinition } from "../../test/view-definition.fixture";
 import { assertConversionToEssence } from "./utils";
 
 describe("Visualization", () => {
@@ -38,7 +38,7 @@ describe("Visualization", () => {
   });
 
   describe("Table", () => {
-    const manifest = TABLE_MANIFEST;
+    const manifest = TABLE_MANIFEST as any as VisualizationManifest;
 
     it("reads table visualization and use default settings", () => {
       assertConversionToEssence(

diff --git a/src/common/view-definitions/version-4/view-definition-4.fixture.ts b/src/common/view-definitions/version-4/view-definition-4.fixture.ts
@@ -19,15 +19,11 @@ import { TOTALS_MANIFEST } from "../../visualization-manifests/totals/totals";
 import { flooredTimeFilterDefinition } from "./filter-definition.fixtures";
 import { ViewDefinition4 } from "./view-definition-4";
 
-const defaults: ViewDefinition4 = {
+export const total: ViewDefinition4 = {
   filters: [flooredTimeFilterDefinition("time", -1, "P1D")],
   splits: [],
-  series: [{ reference: "count" }, { reference: "sum" }],
+  series: [{ reference: "count" }, { reference: "added" }],
   pinnedDimensions: ["string_a"],
   timezone: Timezone.UTC.toString(),
   visualization: TOTALS_MANIFEST.name
 };
-
-export function mockViewDefinition(opts: Partial<ViewDefinition4> = {}): ViewDefinition4 {
-  return { ...defaults, ...opts };
-}