Skip to content

Commit 1605f6b

Browse files
authored
fix(#3430): filtering for numerical dimension properties in data explorer (#3436)
* fix(#3430): Remove unused parameter in DataLakeUtils.ts * fix(#3430): Add method to validate amount of table results in data explorer * fix(#3430): Add e2e test to validate functionality of filter dimension property numbers * fix(#3430): Refactor filter selection panel row and filter operator according to data type * fix(#3430): Numerical values are now handled as expected by filters * fix(#3430): Numerical values are now handled as expected by filters * fix(#3430): Change directory name to pass windows CI builds * fix(#3430): Rename component to meet windows file name length restrictions
1 parent 8517968 commit 1605f6b

28 files changed

+756
-183
lines changed

streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java

+37-13
Original file line numberDiff line numberDiff line change
@@ -34,17 +34,21 @@ public class WhereClauseParams implements IQueryStatement {
3434

3535
private final List<FilterCondition> filterConditions;
3636

37-
private WhereClauseParams(Long startTime,
38-
Long endTime,
39-
String whereConditions) {
37+
private WhereClauseParams(
38+
Long startTime,
39+
Long endTime,
40+
String whereConditions
41+
) {
4042
this(startTime, endTime);
4143
if (whereConditions != null) {
4244
buildConditions(whereConditions);
4345
}
4446
}
4547

46-
private WhereClauseParams(Long startTime,
47-
Long endTime) {
48+
private WhereClauseParams(
49+
Long startTime,
50+
Long endTime
51+
) {
4852
this.filterConditions = new ArrayList<>();
4953
this.buildTimeConditions(startTime, endTime);
5054
}
@@ -56,23 +60,29 @@ private WhereClauseParams(String whereConditions) {
5660
}
5761
}
5862

59-
public static WhereClauseParams from(Long startTime,
60-
Long endTime) {
63+
public static WhereClauseParams from(
64+
Long startTime,
65+
Long endTime
66+
) {
6167
return new WhereClauseParams(startTime, endTime);
6268
}
6369

6470
public static WhereClauseParams from(String whereConditions) {
6571
return new WhereClauseParams(whereConditions);
6672
}
6773

68-
public static WhereClauseParams from(Long startTime,
69-
Long endTime,
70-
String whereConditions) {
74+
public static WhereClauseParams from(
75+
Long startTime,
76+
Long endTime,
77+
String whereConditions
78+
) {
7179
return new WhereClauseParams(startTime, endTime, whereConditions);
7280
}
7381

74-
private void buildTimeConditions(Long startTime,
75-
Long endTime) {
82+
private void buildTimeConditions(
83+
Long startTime,
84+
Long endTime
85+
) {
7686
if (startTime == null) {
7787
this.filterConditions.add(buildTimeBoundary(endTime, LT));
7888
} else if (endTime == null) {
@@ -98,7 +108,9 @@ private void buildConditions(String whereConditions) {
98108
}
99109

100110
private Object returnCondition(String inputCondition) {
101-
if (NumberUtils.isParsable(inputCondition)) {
111+
if (isQuotedString(inputCondition)) {
112+
return removeQuotes(inputCondition);
113+
} else if (NumberUtils.isParsable(inputCondition)) {
102114
return Double.parseDouble(inputCondition);
103115
} else if (isBoolean(inputCondition)) {
104116
return Boolean.parseBoolean(inputCondition);
@@ -107,6 +119,18 @@ private Object returnCondition(String inputCondition) {
107119
}
108120
}
109121

122+
private boolean isQuotedString(String input) {
123+
if (input.startsWith("\"") && input.endsWith("\"")) {
124+
String content = removeQuotes(input);
125+
return NumberUtils.isParsable(content);
126+
}
127+
return false;
128+
}
129+
130+
private String removeQuotes(String input) {
131+
return input.substring(1, input.length() - 1);
132+
}
133+
110134
private boolean isBoolean(String input) {
111135
return "true".equalsIgnoreCase(input) || "false".equalsIgnoreCase(input);
112136
}

ui/angular.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,8 @@
189189
},
190190
"schematics": {
191191
"@schematics/angular:component": {
192-
"style": "scss"
192+
"style": "scss",
193+
"standalone": false
193194
}
194195
},
195196
"cli": {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
timestamp;dimensionKey;v1
2+
1737123058000;1.0;a
3+
1737123059000;2.0;20

ui/cypress/support/utils/connect/ConnectUtils.ts

+16-8
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ export class ConnectUtils {
4848
ConnectEventSchemaUtils.addTimestampProperty();
4949
}
5050

51+
ConnectUtils.configureDimensionProperties(adapterConfiguration);
52+
5153
ConnectEventSchemaUtils.finishEventSchemaConfiguration();
5254

5355
ConnectUtils.startAdapter(
@@ -66,6 +68,20 @@ export class ConnectUtils {
6668

6769
ConnectUtils.configureAdapter(adapterConfiguration);
6870

71+
ConnectUtils.configureDimensionProperties(adapterConfiguration);
72+
73+
if (adapterConfiguration.timestampProperty) {
74+
ConnectEventSchemaUtils.markPropertyAsTimestamp(
75+
adapterConfiguration.timestampProperty,
76+
);
77+
}
78+
79+
ConnectEventSchemaUtils.finishEventSchemaConfiguration();
80+
}
81+
82+
private static configureDimensionProperties(
83+
adapterConfiguration: AdapterInput,
84+
) {
6985
if (adapterConfiguration.dimensionProperties.length > 0) {
7086
adapterConfiguration.dimensionProperties.forEach(
7187
dimensionPropertyName => {
@@ -75,14 +91,6 @@ export class ConnectUtils {
7591
},
7692
);
7793
}
78-
79-
if (adapterConfiguration.timestampProperty) {
80-
ConnectEventSchemaUtils.markPropertyAsTimestamp(
81-
adapterConfiguration.timestampProperty,
82-
);
83-
}
84-
85-
ConnectEventSchemaUtils.finishEventSchemaConfiguration();
8694
}
8795

8896
public static addMachineDataSimulator(

ui/cypress/support/utils/datalake/DataLakeUtils.ts

+43-3
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@ export class DataLakeUtils {
6969

7070
public static loadDataIntoDataLake(
7171
dataSet: string,
72-
wait = true,
7372
format: 'csv' | 'json_array' = 'csv',
7473
) {
7574
// Create adapter with dataset
@@ -182,7 +181,9 @@ export class DataLakeUtils {
182181
}
183182

184183
public static saveDataViewConfiguration() {
185-
cy.dataCy('save-data-view-btn', { timeout: 10000 }).click();
184+
cy.dataCy('save-data-view-btn', { timeout: 10000 }).click({
185+
force: true,
186+
});
186187
}
187188

188189
public static saveDashboardConfiguration() {
@@ -279,6 +280,43 @@ export class DataLakeUtils {
279280
}
280281
}
281282

283+
/**
284+
* This method validates that the defined filter options are available in the UI
285+
* @param expectedFilterOptions
286+
*/
287+
public static validateFilterOptions(
288+
expectedFilterOptions: ('=' | '<' | '<=' | '>=' | '>' | '!=')[],
289+
) {
290+
cy.dataCy('design-panel-data-settings-filter-operator')
291+
.click()
292+
.dataCy('operator-', {}, true)
293+
.should('have.length', expectedFilterOptions.length);
294+
295+
expectedFilterOptions.forEach(option => {
296+
const escapedOption = option.replace(/([=<>!])/g, '\\$1');
297+
cy.dataCy('operator-' + escapedOption).should('be.visible');
298+
});
299+
300+
cy.dataCy('design-panel-data-settings-filter-operator').click({
301+
force: true,
302+
});
303+
}
304+
305+
public static validateAutoCompleteOptions(options: string[]) {
306+
cy.dataCy('design-panel-data-settings-filter-value')
307+
.click({ force: true })
308+
.dataCy('autocomplete-value-', {}, true)
309+
.should('have.length', options.length);
310+
311+
options.forEach(option => {
312+
cy.dataCy('autocomplete-value-' + option).should('be.visible');
313+
});
314+
315+
cy.dataCy('design-panel-data-settings-filter-value').click({
316+
force: true,
317+
});
318+
}
319+
282320
/**
283321
* In the data set panel select all property fields
284322
*/
@@ -310,7 +348,9 @@ export class DataLakeUtils {
310348
}
311349

312350
public static dataConfigRemoveFilter() {
313-
cy.dataCy('design-panel-data-settings-remove-filter').first().click();
351+
cy.dataCy('design-panel-data-settings-remove-filter')
352+
.first()
353+
.click({ force: true });
314354
}
315355

316356
public static clickGroupBy(propertyName: string) {

ui/cypress/support/utils/datalake/DataLakeWidgetTableUtils.ts

+8-4
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,17 @@
1717
*/
1818

1919
export class DataLakeWidgetTableUtils {
20+
public static dataExplorerTableRowTimestamp() {
21+
return cy.dataCy('data-explorer-table-row-timestamp', {
22+
timeout: 10000,
23+
});
24+
}
25+
2026
/**
2127
* Checks how many rows are visible within the table widget in the data explorer
2228
* @param amount of expected rows
2329
*/
24-
public static checkRows(amount: number) {
25-
cy.dataCy('data-explorer-table-row-timestamp', {
26-
timeout: 10000,
27-
}).should('have.length', amount);
30+
public static checkAmountOfRows(amount: number) {
31+
this.dataExplorerTableRowTimestamp().should('have.length', amount);
2832
}
2933
}

ui/cypress/tests/datalake/deleteViewAndDashboard.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
2020
describe('Test Deletion of Data View and Dashboard', () => {
2121
beforeEach('Setup Test', () => {
2222
cy.initStreamPipesTest();
23-
DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv', false);
23+
DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv');
2424
});
2525

2626
it('Perform Test', () => {

ui/cypress/tests/datalake/deleteWidget.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
2020
describe('Test Table View in Data Explorer', () => {
2121
beforeEach('Setup Test', () => {
2222
cy.initStreamPipesTest();
23-
DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv', false);
23+
DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv');
2424
});
2525

2626
it('Perform Test', () => {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
20+
import { DataLakeWidgetTableUtils } from '../../support/utils/datalake/DataLakeWidgetTableUtils';
21+
import { DataLakeFilterConfig } from '../../support/model/DataLakeFilterConfig';
22+
import { AdapterBuilder } from '../../support/builder/AdapterBuilder';
23+
import { ConnectBtns } from '../../support/utils/connect/ConnectBtns';
24+
import { ConnectUtils } from '../../support/utils/connect/ConnectUtils';
25+
import { FileManagementUtils } from '../../support/utils/FileManagementUtils';
26+
27+
describe('Validate that filter works for numerical dimension property', () => {
28+
beforeEach('Setup Test', () => {
29+
cy.initStreamPipesTest();
30+
31+
FileManagementUtils.addFile(
32+
'datalake/filterNumericalStringProperties.csv',
33+
);
34+
const adapterInput = AdapterBuilder.create('File_Stream')
35+
.setName('Test Adapter')
36+
.setTimestampProperty('timestamp')
37+
.addDimensionProperty('dimensionKey')
38+
.setStoreInDataLake()
39+
.setFormat('csv')
40+
.addFormatInput('input', ConnectBtns.csvDelimiter(), ';')
41+
.addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check')
42+
.build();
43+
ConnectUtils.testAdapter(adapterInput);
44+
});
45+
46+
it('Perform Test', () => {
47+
DataLakeUtils.goToDatalake();
48+
DataLakeUtils.createAndEditDataView();
49+
50+
// create table widget and select time range
51+
const startDate = new Date(1737029442000);
52+
const endDate = new Date(1742220659000);
53+
54+
DataLakeUtils.clickOrderBy('descending');
55+
56+
DataLakeUtils.openVisualizationConfig();
57+
DataLakeUtils.selectVisualizationType('Table');
58+
DataLakeUtils.selectTimeRange(startDate, endDate);
59+
cy.wait(1000);
60+
61+
// validate data in table
62+
DataLakeWidgetTableUtils.checkAmountOfRows(2);
63+
64+
// select filter for tag
65+
DataLakeUtils.selectDataConfig();
66+
var filterConfig = new DataLakeFilterConfig('dimensionKey', '1.0', '=');
67+
DataLakeUtils.dataConfigAddFilter(filterConfig);
68+
69+
// validate data in table is filtered
70+
DataLakeWidgetTableUtils.checkAmountOfRows(1);
71+
72+
// remove filter
73+
DataLakeUtils.dataConfigRemoveFilter();
74+
75+
DataLakeUtils.selectDataConfig();
76+
77+
filterConfig = new DataLakeFilterConfig('v1', '20', '=');
78+
DataLakeUtils.dataConfigAddFilter(filterConfig);
79+
80+
// validate data in table is filtered
81+
DataLakeWidgetTableUtils.checkAmountOfRows(1);
82+
83+
// remove filter
84+
DataLakeUtils.dataConfigRemoveFilter();
85+
86+
// validate data again
87+
DataLakeWidgetTableUtils.checkAmountOfRows(2);
88+
});
89+
});

ui/cypress/tests/datalake/missingDataInDataLake.spec.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,13 @@ describe('Test missing properties in data lake', () => {
3434
it('Test table with missing properties', () => {
3535
DataLakeUtils.addDataViewAndTableWidget(dataViewName, 'Persist');
3636

37-
DataLakeWidgetTableUtils.checkRows(5);
37+
DataLakeWidgetTableUtils.checkAmountOfRows(5);
3838

3939
DataLakeUtils.selectDataConfig();
4040
cy.dataCy('data-explorer-ignore-missing-values-checkbox')
4141
.children()
4242
.click();
4343

44-
DataLakeWidgetTableUtils.checkRows(3);
44+
DataLakeWidgetTableUtils.checkAmountOfRows(3);
4545
});
4646
});

ui/cypress/tests/datalake/timeOrderDataView.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import { DataLakeBtns } from '../../support/utils/datalake/DataLakeBtns';
2222
describe('Test Time Order in Data Explorer', () => {
2323
beforeEach('Setup Test', () => {
2424
cy.initStreamPipesTest();
25-
DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv', false);
25+
DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv');
2626
DataLakeUtils.goToDatalake();
2727
DataLakeUtils.createAndEditDataView();
2828
});

ui/cypress/tests/datalake/timeRangeSelectors.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ describe('Test Time Range Selectors in Data Explorer', () => {
4646

4747
before('Setup Tests', () => {
4848
cy.initStreamPipesTest();
49-
DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv', false);
49+
DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv');
5050
});
5151

5252
it('Perform Test', () => {

0 commit comments

Comments
 (0)