Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

split query router #1031

Merged
merged 11 commits into from
Feb 13, 2023
Merged

split query router #1031

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
dbc66fb
string filter queries
adrianmroz-allegro Feb 8, 2023
163d0e5
remove viewDefinitionVersion test
adrianmroz-allegro Feb 8, 2023
73f1572
remove old files
adrianmroz-allegro Feb 8, 2023
9edb81d
number filter query
adrianmroz-allegro Feb 8, 2023
5b1a311
remove meaningles tests
adrianmroz-allegro Feb 8, 2023
0d5e9bd
fix typo
adrianmroz-allegro Feb 9, 2023
5a8ccec
pinboard-query
adrianmroz-allegro Feb 8, 2023
2db9aa7
fix some tests
adrianmroz-allegro Feb 8, 2023
7df1ecb
split query router
adrianmroz-allegro Feb 8, 2023
36e8619
custom error for illegal context access
adrianmroz-allegro Feb 10, 2023
72be4fa
Merge branch 'master' into feature/simplify-query-endpoint
adrianmroz-allegro Feb 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 12 additions & 7 deletions src/client/views/cube-view/api-context.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import { Binary, Nullary, Ternary, Unary } from "../../../common/utils/functiona
import { DEFAULT_VIEW_DEFINITION_VERSION, definitionConverters } from "../../../common/view-definitions";
import { filterDefinitionConverter } from "../../../common/view-definitions/version-4/filter-definition";
import { splitConverter } from "../../../common/view-definitions/version-4/split-definition";

import { Ajax } from "../../utils/ajax/ajax";

export type VisualizationQuery = Unary<Essence, Promise<Dataset>>;
Expand All @@ -49,24 +48,30 @@ export interface ApiContextValue {
pinboardQuery: PinboardQuery;
}

class ApiContextIllegalAccessError extends Error {
constructor() {
super("Attempted to consume ApiContext when there was no Provider");
}
}

export const ApiContext = React.createContext<ApiContextValue>({
get numberFilterQuery(): NumberFilterQuery {
throw new Error("Attempted to consume ApiContext when there was no Provider");
throw new ApiContextIllegalAccessError();
},
get booleanFilterQuery(): BooleanFilterQuery {
throw new Error("Attempted to consume ApiContext when there was no Provider");
throw new ApiContextIllegalAccessError();
},
get visualizationQuery(): VisualizationQuery {
throw new Error("Attempted to consume ApiContext when there was no Provider");
throw new ApiContextIllegalAccessError();
},
get rawDataQuery(): RawDataQuery {
throw new Error("Attempted to consume ApiContext when there was no Provider");
throw new ApiContextIllegalAccessError();
},
get pinboardQuery(): PinboardQuery {
throw new Error("Attempted to consume ApiContext when there was no Provider");
throw new ApiContextIllegalAccessError();
},
get stringFilterQuery(): StringFilterQuery {
throw new Error("Attempted to consume ApiContext when there was no Provider");
throw new ApiContextIllegalAccessError();
}
});

Expand Down
239 changes: 42 additions & 197 deletions src/server/routes/query/query.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,227 +14,72 @@
* limitations under the License.
*/

import { Request, Response, Router } from "express";
import { $, Expression, ply, SortExpression } from "plywood";
import makeGridQuery from "../../../client/visualizations/grid/make-query";
import { Dimension } from "../../../common/models/dimension/dimension";
import { findDimensionByName } from "../../../common/models/dimension/dimensions";
import { NextFunction, Request, Response, Router } from "express";
import { QueryableDataCube } from "../../../common/models/data-cube/queryable-data-cube";
import { Essence } from "../../../common/models/essence/essence";
import { StringFilterClause } from "../../../common/models/filter-clause/filter-clause";
import { LIMIT } from "../../../common/models/raw-data-modal/raw-data-modal";
import { Split } from "../../../common/models/split/split";
import { TimeShiftEnvType } from "../../../common/models/time-shift/time-shift-env";
import { Timekeeper } from "../../../common/models/timekeeper/timekeeper";
import { CANONICAL_LENGTH_ID } from "../../../common/utils/canonical-length/query";
import timeFilterCanonicalLength from "../../../common/utils/canonical-length/time-filter-canonical-length";
import { isNil } from "../../../common/utils/general/general";
import makeQuery from "../../../common/utils/query/visualization-query";
import { DEFAULT_VIEW_DEFINITION_VERSION, definitionConverters } from "../../../common/view-definitions";
import { createEssence } from "../../utils/essence/create-essence";
import { getQueryDecorator } from "../../utils/query-decorator-loader/get-query-decorator";
import { executeQuery } from "../../utils/query/execute-query";
import { asyncHandler } from "../../utils/express/async-handler";
import { AppliedQueryDecorator, getQueryDecorator } from "../../utils/query-decorator-loader/get-query-decorator";
import { handleRequestErrors } from "../../utils/request-errors/handle-request-errors";
import { parseDataCube } from "../../utils/request-params/parse-data-cube";
import { parseDimension } from "../../utils/request-params/parse-dimension";
import {
parseOptionalStringFilterClause,
parseStringFilterClause
} from "../../utils/request-params/parse-filter-clause";
import { parseSplit } from "../../utils/request-params/parse-split";
import { parseViewDefinition } from "../../utils/request-params/parse-view-definition";
import { SettingsManager } from "../../utils/settings-manager/settings-manager";
import booleanFilterRoute from "./routes/boolean-filter";
import numberFilterRoute from "./routes/number-filter";
import pinboardRoute from "./routes/pinboard";
import rawDataRoute from "./routes/raw-data";
import stringFilterRoute from "./routes/string-filter";
import visualizationRoute from "./routes/visualization";

const converter = definitionConverters[DEFAULT_VIEW_DEFINITION_VERSION];

interface QueryRouterContext {
dataCube: QueryableDataCube;
essence: Essence;
decorator: AppliedQueryDecorator;
timekeeper: Timekeeper;
}

export type QueryRouterRequest = Request & {
context?: QueryRouterContext;
};

export function queryRouter(settings: Pick<SettingsManager, "logger" | "getSources" | "appSettings" | "anchorPath" | "getTimekeeper">) {

const router = Router();

router.post("/visualization", async (req: Request, res: Response) => {
router.use(asyncHandler(async (req: Request, res: Response, next: NextFunction) => {
const dataCube = await parseDataCube(req, settings);
const viewDefinition = parseViewDefinition(req);
const essence = createEssence(viewDefinition, converter, dataCube, settings.appSettings);
const decorator = getQueryDecorator(req, dataCube, settings);

function getQuery(essence: Essence, timekeeper: Timekeeper): Expression {
return essence.visualization.name === "grid" ? makeGridQuery(essence, timekeeper) : makeQuery(essence, timekeeper);
}
(req as QueryRouterRequest).context = {
dataCube,
essence,
decorator,
timekeeper: settings.getTimekeeper()
};

try {
const dataCube = await parseDataCube(req, settings);
const viewDefinition = parseViewDefinition(req);
next();
}));

const essence = createEssence(viewDefinition, converter, dataCube, settings.appSettings);
router.post("/visualization", asyncHandler(visualizationRoute));

const query = getQuery(essence, settings.getTimekeeper());
const queryDecorator = getQueryDecorator(req, dataCube, settings);
const result = await executeQuery(dataCube, query, essence.timezone, queryDecorator);
res.json({ result });
} catch (error) {
handleRequestErrors(error, res, settings.logger);
}
});
router.post("/raw-data", asyncHandler(rawDataRoute));

router.post("/raw-data", async (req: Request, res: Response) => {
function getQuery(essence: Essence, timekeeper: Timekeeper): Expression {
const { dataCube } = essence;
const $main = $("main");
const filterExpression = essence
.getEffectiveFilter(timekeeper)
.toExpression(dataCube);
return $main.filter(filterExpression).limit(LIMIT);
}

try {
const dataCube = await parseDataCube(req, settings);
const viewDefinition = parseViewDefinition(req);

const essence = createEssence(viewDefinition, converter, dataCube, settings.appSettings);

const query = getQuery(essence, settings.getTimekeeper());
const queryDecorator = getQueryDecorator(req, dataCube, settings);
const result = await executeQuery(dataCube, query, essence.timezone, queryDecorator);
res.json({ result });
} catch (error) {
handleRequestErrors(error, res, settings.logger);
}
});
router.post("/boolean-filter", asyncHandler(booleanFilterRoute));

router.post("/boolean-filter", async (req: Request, res: Response) => {
function getQuery(essence: Essence, dimension: Dimension, timekeeper: Timekeeper): Expression {
const { dataCube } = essence;
const filterExpression = essence
.getEffectiveFilter(timekeeper, { unfilterDimension: dimension })
.toExpression(dataCube);

return $("main")
.filter(filterExpression)
.split(dimension.expression, dimension.name);
}

try {
const dataCube = await parseDataCube(req, settings);
const viewDefinition = parseViewDefinition(req);
const dimension = parseDimension(req, dataCube);

const essence = createEssence(viewDefinition, converter, dataCube, settings.appSettings);

const query = getQuery(essence, dimension, settings.getTimekeeper());
const queryDecorator = getQueryDecorator(req, dataCube, settings);
const result = await executeQuery(dataCube, query, essence.timezone, queryDecorator);
res.json({ result });
} catch (error) {
handleRequestErrors(error, res, settings.logger);
}
});
router.post("/string-filter", asyncHandler(stringFilterRoute));

router.post("/string-filter", async (req: Request, res: Response) => {

// TODO: expose for UI
const TOP_N = 100;

function getQuery(essence: Essence, clause: StringFilterClause, timekeeper: Timekeeper): Expression {
const { dataCube } = essence;
const { reference: dimensionName } = clause;

const $main = $("main");
const dimension = findDimensionByName(dataCube.dimensions, dimensionName);
const nativeCount = findDimensionByName(dataCube.dimensions, "count");
const measureExpression = nativeCount ? nativeCount.expression : $main.count();

const filter = essence
.changeFilter(essence.filter.setClause(clause))
.getEffectiveFilter(timekeeper).toExpression(dataCube);

return $main
.filter(filter)
.split(dimension.expression, dimension.name)
.apply("MEASURE", measureExpression)
.sort($("MEASURE"), SortExpression.DESCENDING)
.limit(TOP_N);
}

try {
const dataCube = await parseDataCube(req, settings);
const viewDefinition = parseViewDefinition(req);
const clause = parseStringFilterClause(req, dataCube);

const essence = createEssence(viewDefinition, converter, dataCube, settings.appSettings);

const query = getQuery(essence, clause, settings.getTimekeeper());
const queryDecorator = getQueryDecorator(req, dataCube, settings);
const result = await executeQuery(dataCube, query, essence.timezone, queryDecorator);
res.json({ result });
} catch (error) {
handleRequestErrors(error, res, settings.logger);
}
});
router.post("/number-filter", asyncHandler(numberFilterRoute));

router.post("/number-filter", async (req: Request, res: Response) => {

function getQuery(essence: Essence, dimension: Dimension, timekeeper: Timekeeper): Expression {
const { dataCube } = essence;
const filterExpression = essence
.getEffectiveFilter(timekeeper, { unfilterDimension: dimension })
.toExpression(dataCube);
const $main = $("main");
return ply()
.apply("main", $main.filter(filterExpression))
.apply("Min", $main.min($(dimension.name)))
.apply("Max", $main.max($(dimension.name)));
}

try {
const dataCube = await parseDataCube(req, settings);
const viewDefinition = parseViewDefinition(req);
const dimension = parseDimension(req, dataCube);

const essence = createEssence(viewDefinition, converter, dataCube, settings.appSettings);

const query = getQuery(essence, dimension, settings.getTimekeeper());

const queryDecorator = getQueryDecorator(req, dataCube, settings);
const result = await executeQuery(dataCube, query, essence.timezone, queryDecorator);
res.json({ result });
} catch (error) {
handleRequestErrors(error, res, settings.logger);
}
});
router.post("/pinboard", asyncHandler(pinboardRoute));

router.post("/pinboard", async (req: Request, res: Response) => {

function getQuery(essence: Essence, clause: StringFilterClause | null, split: Split, timekeeper: Timekeeper): Expression {
const { dataCube } = essence;

const dimension = findDimensionByName(dataCube.dimensions, split.reference);
const sortSeries = essence.findConcreteSeries(split.sort.reference);
const canonicalLength = timeFilterCanonicalLength(essence, timekeeper);

const filter = essence
.changeFilter(isNil(clause) ? essence.filter.removeClause(split.reference) : essence.filter.setClause(clause))
.getEffectiveFilter(timekeeper)
.toExpression(dataCube);

return $("main")
.filter(filter)
.split(dimension.expression, dimension.name)
.apply(CANONICAL_LENGTH_ID, canonicalLength)
.performAction(sortSeries.plywoodExpression(0, { type: TimeShiftEnvType.CURRENT }))
.performAction(split.sort.toExpression())
.limit(split.limit);
}

try {
const dataCube = await parseDataCube(req, settings);
const viewDefinition = parseViewDefinition(req);
const clause = parseOptionalStringFilterClause(req, dataCube);
const split = parseSplit(req, dataCube);

const essence = createEssence(viewDefinition, converter, dataCube, settings.appSettings);

const query = getQuery(essence, clause, split, settings.getTimekeeper());
const queryDecorator = getQueryDecorator(req, dataCube, settings);
const result = await executeQuery(dataCube, query, essence.timezone, queryDecorator);
res.json({ result });
} catch (error) {
handleRequestErrors(error, res, settings.logger);
}
router.use((error: Error, req: Request, res: Response, next: NextFunction) => {
handleRequestErrors(error, res, settings.logger);
});

return router;
Expand Down
42 changes: 42 additions & 0 deletions src/server/routes/query/routes/boolean-filter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Copyright 2017-2022 Allegro.pl
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Response } from "express";
import { $, Expression } from "plywood";
import { Dimension } from "../../../../common/models/dimension/dimension";
import { Essence } from "../../../../common/models/essence/essence";
import { Timekeeper } from "../../../../common/models/timekeeper/timekeeper";
import { executeQuery } from "../../../utils/query/execute-query";
import { parseDimension } from "../../../utils/request-params/parse-dimension";
import { QueryRouterRequest } from "../query";

function getQuery(essence: Essence, dimension: Dimension, timekeeper: Timekeeper): Expression {
const { dataCube } = essence;
const filterExpression = essence
.getEffectiveFilter(timekeeper, { unfilterDimension: dimension })
.toExpression(dataCube);

return $("main")
.filter(filterExpression)
.split(dimension.expression, dimension.name);
}

export default async function booleanFilterRoute(req: QueryRouterRequest, res: Response) {
const { dataCube, essence, decorator, timekeeper } = req.context;
const dimension = parseDimension(req, dataCube);
const query = getQuery(essence, dimension, timekeeper);
const result = await executeQuery(dataCube, query, essence.timezone, decorator);
res.json({ result });
}
Loading