Skip to content

Commit

Permalink
add multiFilter to run and test labelValues
Browse files Browse the repository at this point in the history
  • Loading branch information
willr3 authored and johnaohara committed May 28, 2024
1 parent 33f0f72 commit a05c6a0
Show file tree
Hide file tree
Showing 10 changed files with 460 additions and 99 deletions.
9 changes: 7 additions & 2 deletions docs/site/content/en/docs/Tutorials/grafana/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,15 +89,20 @@ There are 2 ways to filter:
and `{"version":"1.2.3","txRate":2000}` will add the `txRate=2000` requirement.

> curl --query-param "filter={\"version\":\"1.2.3\",\"txRate\":2000}" <horreum>:/api/test/{id}/labelValues
Grafana offers a multi-select option for variables. This sends the options as an array when using `json` encoding.
Horreum will default to looking for a label with an array value instead of any value in the array.
Adding the `multiFilter=true` query parameter allows Horreum to also look for any value in the array and supports Grafana mulit-select.

> curl --query-param "multiFilter=true" --query-param "filter={\"count\":[1,2,4,8]}" <horreum>:/api/test/{id}/labelValues
2. provide a json path (an extractor path from labels) that needs to evaluate to true

For example, if `count` is a label we can pass in `$.count ? (@ > 10 && @ < 20)` to only include datasets where count is between 10 and 20.

> curl --query-param "filter=\"$.count ? (@ > 10 && @ < 20)\"" <horreum>:/api/test/{id}/labelValues
We set the `filter` parameter by editing the Query for the grafana panel but it will depend .
We set the `filter` parameter by editing the Query for the grafana panel.

{{% imgproc json_api_panel_filter Fit "865x331" %}}
Define filter for query
Expand Down
14 changes: 14 additions & 0 deletions docs/site/content/en/openapi/openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1043,6 +1043,13 @@ paths:
multiple:
description: excluding multiple labels
value: "id,count"
- name: multiFilter
in: query
description: enable filtering for multiple values with an array of values
schema:
default: false
type: boolean
example: true
responses:
"200":
description: label Values
Expand Down Expand Up @@ -2155,6 +2162,13 @@ paths:
multiple:
description: excluding multiple labels
value: "id,count"
- name: multiFilter
in: query
description: enable filtering for multiple values with an array of values
schema:
default: false
type: boolean
example: true
responses:
"200":
description: OK
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ Object getData(@PathParam("id") int id,
examples = {
@ExampleObject(name="single", value="id", description = "excluding a single label"),
@ExampleObject(name="multiple", value="id,count", description = "excluding multiple labels")
})
}),
@Parameter(name = "multiFilter", description = "enable filtering for multiple values with an array of values", example = "true")
})
@APIResponses(
value = {
Expand All @@ -145,7 +146,8 @@ List<ExportedLabelValues> labelValues(
@QueryParam("limit") @DefaultValue(""+Integer.MAX_VALUE) int limit,
@QueryParam("page") @DefaultValue("0") int page,
@QueryParam("include") @Separator(",") List<String> include,
@QueryParam("exclude") @Separator(",") List<String> exclude);
@QueryParam("exclude") @Separator(",") List<String> exclude,
@QueryParam("multiFilter") @DefaultValue("false") boolean multiFilter);

@GET
@Path("{id}/metadata")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,8 @@ void updateNotifications(@PathParam("id") int id,
examples = {
@ExampleObject(name="single", value="id", description = "excluding a single label"),
@ExampleObject(name="multiple", value="id,count", description = "excluding multiple labels")
})
}),
@Parameter(name = "multiFilter", description = "enable filtering for multiple values with an array of values", example = "true")
})
@APIResponses(
value = { @APIResponse( responseCode = "200",
Expand All @@ -247,7 +248,8 @@ List<ExportedLabelValues> labelValues(
@QueryParam("limit") @DefaultValue(""+Integer.MAX_VALUE) int limit,
@QueryParam("page") @DefaultValue("0") int page,
@QueryParam("include") @Separator(",") List<String> include,
@QueryParam("exclude") @Separator(",") List<String> exclude);
@QueryParam("exclude") @Separator(",") List<String> exclude,
@QueryParam("multiFilter") @DefaultValue("false") boolean multiFilter);

@POST
@Consumes(MediaType.APPLICATION_JSON)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
package io.hyperfoil.tools.horreum.hibernate;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import io.hyperfoil.tools.horreum.svc.Util;
import org.hibernate.HibernateException;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.type.CustomType;
import org.hibernate.type.SqlTypes;
import org.hibernate.type.spi.TypeConfiguration;
import org.hibernate.usertype.UserType;

import java.io.Serializable;
import java.sql.*;
import java.util.*;

import static java.lang.String.format;

public class JsonbSetType implements UserType<ArrayNode> {


public static final CustomType INSTANCE = new CustomType<>(new JsonbSetType(), new TypeConfiguration());

@Override
public int getSqlType() {
return SqlTypes.ARRAY;
}

@Override
public Class<ArrayNode> returnedClass() {
return ArrayNode.class;
}

@Override
public boolean equals(ArrayNode x, ArrayNode y) {
return x.equals(y);
}

@Override
public int hashCode(ArrayNode x) {
return Objects.hashCode(x);
}

@Override
public ArrayNode nullSafeGet(ResultSet rs, int position, SharedSessionContractImplementor session, Object owner)
throws SQLException {
if(rs.wasNull())
return null;
Array array = rs.getArray(position);
if (array == null) {
return null;
}
try {
String[] raw = (String[]) array.getArray();
ArrayNode rtrn = JsonNodeFactory.instance.arrayNode();
for(int i=0; i<raw.length; i++){
rtrn.add(Util.toJsonNode(raw[i]));
}
return rtrn;
} catch (final Exception ex) {
throw new RuntimeException("Failed to convert ResultSet to json array: " + ex.getMessage(), ex);
}
}

@Override
public void nullSafeSet(PreparedStatement ps, ArrayNode value, int index, SharedSessionContractImplementor session)
throws SQLException {
if (value == null) {
ps.setNull(index, Types.ARRAY);
return;
}
try {
Set<String> str = new HashSet<>();
value.forEach(v->str.add(v.toString()));
Array array = ps.getConnection().createArrayOf("jsonb", str.toArray());
ps.setObject(index, array);
} catch (final Exception ex) {
throw new RuntimeException(format("Failed to convert JSON to String: %s", ex.getMessage()), ex);
}
}

@Override
public ArrayNode deepCopy(ArrayNode value) throws HibernateException {
if (value == null) {
return null;
}
try {
return (ArrayNode)new ObjectMapper().readTree(value.toString());
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}

@Override
public boolean isMutable() {
return true;
}

@Override
public Serializable disassemble(ArrayNode value) throws HibernateException {
return value.toString();
}

@Override
public ArrayNode assemble(Serializable cached, Object owner) throws HibernateException {
try {
return (ArrayNode)new ObjectMapper().readTree(cached.toString());
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}

public String getName(){ return "jsonb-any";}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import io.hyperfoil.tools.horreum.bus.AsyncEventChannels;
import io.hyperfoil.tools.horreum.entity.alerting.DataPointDAO;
import io.hyperfoil.tools.horreum.hibernate.JsonBinaryType;
import io.hyperfoil.tools.horreum.hibernate.JsonbSetType;
import io.hyperfoil.tools.horreum.mapper.DatasetMapper;
import jakarta.annotation.security.PermitAll;
import jakarta.annotation.security.RolesAllowed;
Expand Down Expand Up @@ -274,37 +275,27 @@ public Object getData(int id, String token, String schemaUri) {
//this is nearly identical to TestServiceImpl.labelValues (except the return object)
//this reads from the dataset table but provides data specific to the run...
@Override
public List<ExportedLabelValues> labelValues(int runId, String filter, String sort, String direction, int limit, int page, List<String> include, List<String> exclude){
public List<ExportedLabelValues> labelValues(int runId, String filter, String sort, String direction, int limit, int page, List<String> include, List<String> exclude, boolean multiFilter){
List<ExportedLabelValues> rtrn = new ArrayList<>();
Run run = getRun(runId,null);
if(run == null){
throw ServiceException.serverError("Cannot find run "+runId);
}
Object filterObject = Util.getFilterObject(filter);
String filterSql = "";
if(filterObject instanceof JsonNode && ((JsonNode)filterObject).getNodeType() == JsonNodeType.OBJECT){
filterSql = "WHERE "+TestServiceImpl.LABEL_VALUES_FILTER_CONTAINS_JSON;
}else {
Util.CheckResult jsonpathResult = Util.castCheck(filter,"jsonpath",em);
if(jsonpathResult.ok()) {
filterSql = "WHERE "+TestServiceImpl.LABEL_VALUES_FILTER_MATCHES_NOT_NULL;
} else {
if(filter!=null && filter.startsWith("{") && filter.endsWith("}")) {
Util.CheckResult jsonbResult = Util.castCheck(filter, "jsonb", em);
if (!jsonbResult.ok()) {
//we expect this error (because filterObject is not JsonNode
} else {
//this would be a surprise and quite a problem
}
} else {
//how do we report back invalid jsonpath
}
}

TestServiceImpl.FilterDef filterDef = TestServiceImpl.getFilterDef(filter,null,null,multiFilter,(str)->
labelValues(runId,str,sort,direction,limit,page,include,exclude,false),em);

String filterSql = filterDef.sql();
if(filterDef.filterObject()!=null){
filterObject = filterDef.filterObject();
}

if(filterSql.isBlank() && filter != null && !filter.isBlank()){
//TODO there was an error with the filter, do we return that info to the user?
}
String orderSql = "";

String orderDirection = direction.equalsIgnoreCase("ascending") ? "ASC" : "DESC";
if(!sort.isBlank()){
Util.CheckResult jsonpathResult = Util.castCheck(sort, "jsonpath", em);
Expand All @@ -315,12 +306,13 @@ public List<ExportedLabelValues> labelValues(int runId, String filter, String so
}
}
String includeExcludeSql = "";
List<String> mutableInclude = new ArrayList<>(include);

if (include!=null && !include.isEmpty()) {
if (exclude != null && !exclude.isEmpty()) {
include = new ArrayList<>(include);
include.removeAll(exclude);
mutableInclude.removeAll(exclude);
}
if (!include.isEmpty()) {
if (!mutableInclude.isEmpty()) {
includeExcludeSql = " AND label.name in :include";
}
}
Expand Down Expand Up @@ -349,12 +341,21 @@ SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE la
if(!filterSql.isEmpty()) {
if (filterSql.contains(TestServiceImpl.LABEL_VALUES_FILTER_CONTAINS_JSON)) {
query.setParameter("filter", filterObject, JsonBinaryType.INSTANCE);
} else {
} else if (filterSql.contains(TestServiceImpl.LABEL_VALUES_FILTER_MATCHES_NOT_NULL)){
query.setParameter("filter", filter);
}
}
if(!filterDef.multis().isEmpty() && filterDef.filterObject()!=null){
ObjectNode fullFilterObject = (ObjectNode) Util.getFilterObject(filter);
for(int i=0; i<filterDef.multis().size(); i++){
String key = filterDef.multis().get(i);
ArrayNode value = (ArrayNode) fullFilterObject.get(key);
query.setParameter("key"+i,"$."+key);
query.setParameter("value"+i,value, JsonbSetType.INSTANCE);
}
}
if(includeExcludeSql.contains(":include")){
query.setParameter("include",include);
query.setParameter("include",mutableInclude);
}else if (includeExcludeSql.contains(":exclude")){
query.setParameter("exclude",exclude);
}
Expand All @@ -369,7 +370,6 @@ SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE la
.addScalar("datasetId",Integer.class)
.addScalar("start", StandardBasicTypes.INSTANT)
.addScalar("stop", StandardBasicTypes.INSTANT);

//casting because type inference cannot detect there will be two scalars in the result
//TODO replace this with strictly typed entries
((List<Object[]>) query.getResultList()).forEach(objects->{
Expand Down
Loading

0 comments on commit a05c6a0

Please sign in to comment.