Skip to content

Commit

Permalink
[add] Subqueries for array props
Browse files Browse the repository at this point in the history
  • Loading branch information
@amivanoff
amivanoff committed Nov 10, 2021
1 parent e389e42 commit 00ac0dc
Show file tree
Hide file tree
Showing 7 changed files with 360 additions and 179 deletions.
3 changes: 2 additions & 1 deletion .vscode/settings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,5 +80,6 @@
"cSpell.language": "en,ru",
"terminal.integrated.scrollback": 10000,
"jest.jestCommandLine": "yarn test",
"jest.autoRun": "off"
"jest.autoRun": "off",
"jest.enableCodeLens": false
}
12 changes: 6 additions & 6 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@agentlab/sparql-jsld-client",
"version": "5.0.1",
"version": "5.0.2",
"description": "SPARQL JSON Schema Linked Data Client",
"license": "GPL-3.0",
"author": "Alexey Ivanov <amivanoff@gmail.com>",
Expand Down Expand Up @@ -59,7 +59,7 @@
"jsonld": ">=5.2.0",
"lodash-es": ">=4.17.21",
"mobx": ">=6.3.6",
"mobx-state-tree": ">=5.0.4",
"mobx-state-tree": ">=5.0.5",
"moment": ">=2.29.1",
"rdf-literal": ">=1.3.0",
"sparqljs": ">=3.5.1",
Expand All @@ -77,7 +77,7 @@
"jsonld": "^5.2.0",
"lodash-es": "^4.17.21",
"mobx": "^6.3.6",
"mobx-state-tree": "^5.0.4",
"mobx-state-tree": "^5.0.5",
"moment": "^2.29.1",
"rdf-literal": "^1.3.0",
"sparqljs": "^3.5.1",
Expand All @@ -93,9 +93,9 @@
"@rollup/plugin-typescript": "^8.3.0",
"@types/is-url": "^1.2.30",
"@types/jest": "^27.0.2",
"@types/node": "^16.11.6",
"@typescript-eslint/eslint-plugin": "^5.3.0",
"@typescript-eslint/parser": "^5.3.0",
"@types/node": "^16.11.7",
"@typescript-eslint/eslint-plugin": "^5.3.1",
"@typescript-eslint/parser": "^5.3.1",
"custom-env": "^2.0.1",
"babel-loader": "^8.2.3",
"cross-env": "^7.0.3",
Expand Down
5 changes: 4 additions & 1 deletion src/SparqlGen.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,7 @@ export interface EntConstrInternal extends EntConstrData {
schemaPropsWithArrays: any;
conditionsWithoutArrays: JsObject;
conditionsWithArrays: JsObject;
props: JsObject;
/**
* Outgoing References dictionary
* ConditionProp: EntConstrNumberInArray
Expand All @@ -270,6 +271,7 @@ export interface EntConstrInternal extends EntConstrData {
*/
relatedFrom: { [s: string]: number };
bindsVars: JsObject;
subEntConstr?: EntConstrInternal;
}

export function unscreenIds(data: any | undefined) {
Expand Down Expand Up @@ -674,7 +676,7 @@ export function getConditionalTriple(
return getTripleWithPredOrPath(subj, propUri, value, prefixes);
}

function getTripleWithPredOrPath(subj: any, propUri: string | string[], value: any, prefixes: JsStrObj): Quad {
export function getTripleWithPredOrPath(subj: any, propUri: string | string[], value: any, prefixes: JsStrObj): Quad {
if (isArray(propUri)) {
return triple(
subj,
Expand Down Expand Up @@ -1185,6 +1187,7 @@ export function getInternalCollConstrs(
schemaPropsWithArrays: {},
conditionsWithoutArrays: {},
conditionsWithArrays: {},
props: {},
relatedTo: {},
relatedFrom: {},
bindsVars: {},
Expand Down
247 changes: 138 additions & 109 deletions src/SparqlGenSelect.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ import {
getSchemaPropType,
deAbbreviateIri,
localUrn,
getSchemaPropUri,
getTripleWithPredOrPath,
} from './SparqlGen';

export async function selectObjectsQuery(collConstrJs2: ICollConstrJsOpt, nsJs: any, client: SparqlClient) {
Expand Down Expand Up @@ -434,6 +436,66 @@ export async function constructObjectsQuery(
const objects: JsObject[] = await jsonLdToObjects(results, entConstrs);
return objects;
}
//TODO: replace getWhereVar
export function getWhereProps(
entConstr: EntConstrInternal,
requireOptional = false,
): { bgps: Quad[]; options: Quad[] } {
const bgps: Quad[] = [];
const options: Quad[] = [];
Object.keys(entConstr.props).forEach((key) => {
// filter @id, @type,...
if (!key.startsWith('@')) {
const propUri = getSchemaPropUri(entConstr.schema, key);
const varName = entConstr.props2vars[key];
if (propUri && varName) {
const option = getTripleWithPredOrPath(entConstr.subj, propUri, variable(varName), entConstr.prefixes);
if ((entConstr.schema.required && entConstr.schema.required.includes(key)) || requireOptional) {
bgps.push(option);
} else {
options.push(option);
}
}
}
});
return { bgps, options };
}

function processEntConstr(entConstr: EntConstrInternal, index: number, addType = true) {
const { bgps, options } = getWhereProps(entConstr);
entConstr.qCond = processConditions(entConstr, entConstr.conditions);
// make all conditions mandatory
entConstr.qCond.bgps = [...entConstr.qCond.bgps, ...entConstr.qCond.options];
entConstr.qCond.options = [];

if (entConstr.resolveType) {
entConstr.qTypeFilters = genSuperTypesFilter(entConstr, index);
entConstr.qTypeConds = [...entConstr.qTypeConds, ...genTypeCondition(entConstr), entConstr.qTypeFilters[0]];
} else if (addType) {
entConstr.qTypeConds = genTypeCondition(entConstr);
}

entConstr.query.bgps = [...entConstr.qTypeConds, ...entConstr.query.bgps, ...entConstr.qCond.bgps, ...bgps];
entConstr.query.options = [...entConstr.query.options, ...entConstr.qCond.options, ...options];
entConstr.query.filters = [...entConstr.query.filters, ...entConstr.qTypeFilters, ...entConstr.qCond.filters];
entConstr.query.binds = entConstr.qCond.binds;

entConstr.bindsVars = entConstr.qCond.bindsVars;
const bindsVarsTriples = Object.keys(entConstr.qCond.bindsVars).map((key) => {
const varName = entConstr.qCond.bindsVars[key];
return triple(entConstr.subj, namedNode(localUrn(key)), variable(varName));
});
const { bgps: bgps2 } = getWhereProps(entConstr, true);

entConstr.query.templates = fixPropPath([
...entConstr.qTypeConds,
...entConstr.query.templates,
...entConstr.qCond.bgps, // conditionsBeforeSeparation
...entConstr.qCond.options, // conditionsBeforeSeparation
...bgps2,
...bindsVarsTriples,
]);
}

/**
*
Expand All @@ -446,22 +508,13 @@ function constructQueryFromEntConstrs(entConstrs: EntConstrInternal[], collConst
entConstrs.forEach((entConstr, index) => {
if (entConstr.schema.properties) {
if (entConstr.variables) {
//TODO: it could be an array property conflicted with LIMIT
copyUniqueObjectProps(entConstr.query.variables, entConstr.variables);
} else {
const entConstrJs = collConstrJs.entConstrs[index];
if (entConstrJs.limit) {
copyUniqueObjectProps(
entConstr.query.variables,
copyObjectPropsWithRenameOrFilter(entConstr.schemaPropsWithoutArrays, entConstr.ignoredProperties),
);
} else {
//copy the rest
copyUniqueObjectProps(
entConstr.query.variables,
copyObjectPropsWithRenameOrFilter(entConstr.schema.properties, entConstr.ignoredProperties),
);
}
//copy the rest
copyUniqueObjectProps(
entConstr.query.variables,
copyObjectPropsWithRenameOrFilter(entConstr.schema.properties, entConstr.ignoredProperties),
);
}
}
if (entConstr.resolveType) {
Expand All @@ -472,88 +525,34 @@ function constructQueryFromEntConstrs(entConstrs: EntConstrInternal[], collConst
}
renumerateEntConstrVariables(entConstr, index);
});
// form partial query patterns (bgps, options, filters, binds)
entConstrs.forEach((entConstr, index) => {
const { bgps, options } = getWhereVar(entConstr);
let isSubqueries = false;
// if LIMIT extract 1:1 cardinality sub-query in each LIMITed entConstr
entConstrs = entConstrs.map((entConstr, index) => {
const entConstrJs = collConstrJs.entConstrs[index];
//let conditions: any = {};
if (entConstrJs.limit) {
//TODO: In case of array props, make subquery for every proptrty from entConstr.schemaPropsWithoutArrays
entConstr.qCond = processConditions(entConstr, entConstr.conditionsWithoutArrays, true);
//if array conditions are also relations, push-down bgps
Object.entries(entConstr.conditionsWithArrays).forEach((c) => {
const toEntConstrIndex: number | undefined = entConstr.relatedTo[c[0]];
if (toEntConstrIndex) {
const toEntConstr = entConstrs[toEntConstrIndex];
const conditionWithArrays = processConditions(entConstr, { [c[0]]: c[1] }, true);
conditionWithArrays.bgps = fixPropPath(conditionWithArrays.bgps);
toEntConstr.query.bgps = [...toEntConstr.query.bgps, ...conditionWithArrays.bgps];
toEntConstr.query.templates = [...toEntConstr.query.templates, ...conditionWithArrays.bgps];
}
});
} else {
//entConstr.qCond = processConditions(entConstr, entConstr.conditions);
entConstr.qCond = processConditions(entConstr, entConstr.conditions);
// push-down relation bgp to the child (object) group but preserve it in parent (subject) template (for style)
if (entConstrs.length > 1) {
const condBgpsSeparation = separateReferencedQuads(entConstr.qCond.bgps);
entConstr.qCond.bgps = condBgpsSeparation.nonRefs;
condBgpsSeparation.refs.forEach((r) => {
// if child entity (very rough)
if (r.index > index) {
const toEntConstr = entConstrs[r.index];
toEntConstr.query.bgps = [...toEntConstr.query.bgps, r.quad];
entConstr.query.templates = [...entConstr.query.templates, r.quad];
} else {
entConstr.qCond.bgps = [...entConstr.qCond.bgps, r.quad];
}
});
const condOptsSeparation = separateReferencedQuads(entConstr.qCond.options);
entConstr.qCond.options = condOptsSeparation.nonRefs;
// make optional condition required in optional block
condOptsSeparation.refs.forEach((r) => {
// if child entity (very rough)
if (r.index > index) {
const toEntConstr = entConstrs[r.index];
toEntConstr.query.bgps = [...toEntConstr.query.bgps, r.quad];
entConstr.query.templates = [...entConstr.query.templates, r.quad];
} else {
entConstr.qCond.options = [...entConstr.qCond.options, r.quad];
}
});
}
if (!entConstrJs.limit || Object.keys(entConstr.schemaPropsWithoutArrays).length === 0) {
entConstr.props = entConstr.query.variables;
processEntConstr(entConstr, index);
return entConstr;
}
// make all conditions mandatory
entConstr.qCond.bgps = [...entConstr.qCond.bgps, ...entConstr.qCond.options];
entConstr.qCond.options = [];

if (entConstr.resolveType) {
entConstr.qTypeFilters = genSuperTypesFilter(entConstr, index);
entConstr.qTypeConds = [...entConstr.qTypeConds, ...genTypeCondition(entConstr), entConstr.qTypeFilters[0]];
} else {
entConstr.qTypeConds = genTypeCondition(entConstr);
}

entConstr.query.bgps = [...entConstr.qTypeConds, ...entConstr.query.bgps, ...entConstr.qCond.bgps, ...bgps];
entConstr.query.options = [...entConstr.query.options, ...entConstr.qCond.options, ...options];
entConstr.query.filters = [...entConstr.query.filters, ...entConstr.qTypeFilters, ...entConstr.qCond.filters];
entConstr.query.binds = entConstr.qCond.binds;

entConstr.bindsVars = entConstr.qCond.bindsVars;
const bindsVarsTriples = Object.keys(entConstr.qCond.bindsVars).map((key) => {
const varName = entConstr.qCond.bindsVars[key];
return triple(entConstr.subj, namedNode(localUrn(key)), variable(varName));
});
const { bgps: bgps2 } = getWhereVar(entConstr, true);

entConstr.query.templates = [
...entConstr.qTypeConds,
...entConstr.query.templates,
...entConstr.qCond.bgps, // conditionsBeforeSeparation
...entConstr.qCond.options, // conditionsBeforeSeparation
...bgps2,
...bindsVarsTriples,
];
isSubqueries = true;
const subEntConstr: EntConstrInternal = cloneDeep(entConstr);
subEntConstr.query.variables = {};
copyUniqueObjectProps(
subEntConstr.query.variables,
copyObjectPropsWithRenameOrFilter(entConstr.schemaPropsWithoutArrays, entConstr.ignoredProperties),
);
subEntConstr.props = subEntConstr.query.variables;
subEntConstr.conditions = entConstr.conditionsWithoutArrays;
copyUniqueObjectProps(
entConstr.props,
copyObjectPropsWithRenameOrFilter(entConstr.schemaPropsWithArrays, entConstr.ignoredProperties),
);
entConstr.conditions = entConstr.conditionsWithArrays;
entConstr.resolveType = false;
entConstr.subEntConstr = subEntConstr;
processEntConstr(subEntConstr, index);
processEntConstr(entConstr, index, false);
return entConstr;
});

const query: ConstructQuery & {
Expand All @@ -570,21 +569,36 @@ function constructQueryFromEntConstrs(entConstrs: EntConstrInternal[], collConst
template: [],
};
let allOptions: any[] = [];
let isSubqueries = false;

entConstrs.forEach((entConstr, index) => {
const entConstrJs = collConstrJs.entConstrs[index];
if (entConstrJs.limit) {
if (entConstr.subEntConstr) {
// LIMIT exists, use SELECT subquery
// array fields breaks LIMIT due to results denormalization
isSubqueries = true;
const { variables, where, options } = selectQueryFromEntConstr(entConstr);
let where2: any[] = [...where, ...options];
const {
variables: variables2,
where: where2,
options: options2,
} = selectQueryFromEntConstr(entConstr.subEntConstr);
let whereAll2: any[] = [...where2, ...options2];
const { variables: variables1, where: where1, options: options1 } = selectQueryFromEntConstr(entConstr);
let whereAll1: any[] = [...where1, ...options1];
if (entConstrJs.service) {
where2 = [
whereAll2 = [
{
type: 'service',
patterns: where2,
patterns: whereAll2,
name: {
termType: 'NamedNode',
value: entConstrJs.service,
},
silent: false,
},
];
whereAll1 = [
{
type: 'service',
patterns: whereAll1,
name: {
termType: 'NamedNode',
value: entConstrJs.service,
Expand All @@ -593,24 +607,39 @@ function constructQueryFromEntConstrs(entConstrs: EntConstrInternal[], collConst
},
];
}
const subQuery: SelectQuery = {
const subQuery2: SelectQuery = {
type: 'query',
queryType: 'SELECT',
prefixes: {},
variables,
where: where2,
variables: variables2,
where: whereAll2,
};
if (entConstrJs.orderBy) subQuery2.order = entConstrJs.orderBy;
if (entConstrJs.limit) subQuery2.limit = entConstrJs.limit;
if (entConstrJs.offset) subQuery2.offset = entConstrJs.offset;
if (entConstrJs.distinct) subQuery2.distinct = entConstrJs.distinct;
const subQuery1: SelectQuery = {
type: 'query',
queryType: 'SELECT',
prefixes: {},
variables: variables1,
//where: whereAll1,
where: [
{
type: 'group',
patterns: [subQuery2],
},
...whereAll1,
],
};
if (entConstrJs.orderBy) subQuery.order = entConstrJs.orderBy;
if (entConstrJs.limit) subQuery.limit = entConstrJs.limit;
if (entConstrJs.offset) subQuery.offset = entConstrJs.offset;
if (entConstrJs.distinct) subQuery.distinct = entConstrJs.distinct;
query.where = [
...(query.where || []),
{
type: 'group',
patterns: [subQuery],
patterns: [subQuery1],
},
];
entConstr.query.templates = [...entConstr.subEntConstr.query.templates, ...entConstr.query.templates];
} else {
// no LIMIT, proceed with CONSTRUCT (without a subquery)
entConstr.query.bgps =
Expand Down
Loading

0 comments on commit 00ac0dc

Please sign in to comment.