add include and exclude to test and run service labelValues endpoint

docs/site/content/en/docs/Tutorials/grafana/index.md

@@ -103,3 +103,24 @@ We set the `filter` parameter by editing the Query for the grafana panel but it
 Define filter for query
 {{% /imgproc %}}
 
+## Filtering labels
+
+Another common consideration is the amount of data in the `/labelValues` response. Tests with lots of labels, or labels that produce a lot of data,
+can see the `/labelValues` transfer json size grow well beyond what they need for a particular integration. Horreum has the `include` and `exclude`
+query parameter options on the `/labelValues` endpoint.
+
+### Include
+Adding `include=foo` to the `/labelValues` endpoint query tells Horreum to only include the `foo` label and its value in the `values` part of the 
+`/labelValues` response. You can specify multiple labels with `incude=foo&include=bar` or `include=foo,bar` using url encoding or with curl:
+```bash
+curl --query-param "include=foo" --query-param "include=bar" ...
+```
+
+Note: any `include` that is also mentioned in `exclude` will not be part of the response `values`
+
+### Exclude
+This functions similar to `include` except that it removes a label name from the response `values` field for the `/labelValues` endpoint. This filter
+option leaves all other labels in the `values` field.
+If a user specifies both `include` and `exclude` then the response will only contain the `include` label names that are not also in `exclude`. If all
+`include` are also in `exclude` then the `exclude` takes priority and the response will contain all labels that are not mentioned in `exclude`.
+Horreum uses this default behavior to avoid sending any data that is explicitly excluded. 

docs/site/content/en/openapi/openapi.yaml

@@ -1012,6 +1012,22 @@ paths:
  default: 0
  type: integer
  example: 2
+ - name: include
+ in: query
+ description: name of a label to include in the result
+ schema:
+ type: array
+ items:
+ type: string
+ example: id
+ - name: exclude
+ in: query
+ description: name of a label to exclude from the result
+ schema:
+ type: array
+ items:
+ type: string
+ example: id
  responses:
  "200":
  description: label Values
@@ -2008,7 +2024,7 @@ paths:
  tags:
  - Test
  description: List all Label Values for a Test
- operationId: listLabelValues
+ operationId: labelValues
  parameters:
  - name: id
  in: path
@@ -2090,6 +2106,22 @@ paths:
  default: 0
  type: integer
  example: 2
+ - name: include
+ in: query
+ description: name of a label to include in the result
+ schema:
+ type: array
+ items:
+ type: string
+ example: id
+ - name: exclude
+ in: query
+ description: name of a label to exclude from the result
+ schema:
+ type: array
+ items:
+ type: string
+ example: id
  responses:
  "200":
  description: OK

horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/services/RunService.java

@@ -31,6 +31,7 @@
 import org.eclipse.microprofile.openapi.annotations.responses.APIResponses;
 import org.eclipse.microprofile.openapi.annotations.tags.Tag;
 import org.jboss.resteasy.reactive.RestForm;
+import org.jboss.resteasy.reactive.Separator;
 import org.jboss.resteasy.reactive.multipart.FileUpload;
 
 @Path("/api/run")
@@ -110,7 +111,17 @@ Object getData(@PathParam("id") int id,
  @Parameter(name = "sort", description = "label name for sorting"),
  @Parameter(name = "direction",description = "either Ascending or Descending",example="count"),
  @Parameter(name = "limit",description = "the maximum number of results to include",example="10"),
- @Parameter(name = "page",description = "which page to skip to when using a limit",example="2")
+ @Parameter(name = "page",description = "which page to skip to when using a limit",example="2"),
+ @Parameter(name = "include", description = "label name(s) to include in the result as scalar or comma separated",
+ examples = {
+ @ExampleObject(name="single", value="id", description = "including a single label"),
+ @ExampleObject(name="multiple", value="id,count", description = "including multiple labels")
+ }),
+ @Parameter(name = "exclude", description = "label name(s) to exclude from the result as scalar or comma separated",
+ examples = {
+ @ExampleObject(name="single", value="id", description = "excluding a single label"),
+ @ExampleObject(name="multiple", value="id,count", description = "excluding multiple labels")
+ })
  })
  @APIResponses(
  value = {
@@ -132,7 +143,9 @@ List<ExportedLabelValues> labelValues(
  @QueryParam("sort") @DefaultValue("") String sort,
  @QueryParam("direction") @DefaultValue("Ascending") String direction,
  @QueryParam("limit") @DefaultValue(""+Integer.MAX_VALUE) int limit,
- @QueryParam("page") @DefaultValue("0") int page);
+ @QueryParam("page") @DefaultValue("0") int page,
+ @QueryParam("include") @Separator(",") List<String> include,
+ @QueryParam("exclude") @Separator(",") List<String> exclude);
 
  @GET
  @Path("{id}/metadata")

horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/services/TestService.java

@@ -35,6 +35,7 @@
 import org.eclipse.microprofile.openapi.annotations.responses.APIResponseSchema;
 import org.eclipse.microprofile.openapi.annotations.responses.APIResponses;
 import org.eclipse.microprofile.openapi.annotations.tags.Tag;
+import org.jboss.resteasy.reactive.Separator;
 
 import java.util.Collection;
 import java.util.List;
@@ -216,15 +217,25 @@ void updateNotifications(@PathParam("id") int id,
  @Parameter(name = "sort", description = "json path to sortable value or start or stop for sorting by time",example = "$.label or start or stop"),
  @Parameter(name = "direction",description = "either Ascending or Descending",example="count"),
  @Parameter(name = "limit",description = "the maximum number of results to include",example="10"),
- @Parameter(name = "page",description = "which page to skip to when using a limit",example="2")
+ @Parameter(name = "page",description = "which page to skip to when using a limit",example="2"),
+ @Parameter(name = "include", description = "label name(s) to include in the result as scalar or comma separated",
+ examples = {
+ @ExampleObject(name="single", value="id", description = "including a single label"),
+ @ExampleObject(name="multiple", value="id,count", description = "including multiple labels")
+ }),
+ @Parameter(name = "exclude", description = "label name(s) to exclude from the result as scalar or comma separated",
+ examples = {
+ @ExampleObject(name="single", value="id", description = "excluding a single label"),
+ @ExampleObject(name="multiple", value="id,count", description = "excluding multiple labels")
+ })
  })
  @APIResponses(
  value = { @APIResponse( responseCode = "200",
  content = {
  @Content ( schema = @Schema(type = SchemaType.ARRAY, implementation = ExportedLabelValues.class)) }
  )}
  )
- List<ExportedLabelValues> listLabelValues(
+ List<ExportedLabelValues> labelValues(
  @PathParam("id") int testId,
  @QueryParam("filter") @DefaultValue("{}") String filter,
  @QueryParam("before") @DefaultValue("") String before,
@@ -234,7 +245,9 @@ List<ExportedLabelValues> listLabelValues(
  @QueryParam("sort") @DefaultValue("") String sort,
  @QueryParam("direction") @DefaultValue("Ascending") String direction,
  @QueryParam("limit") @DefaultValue(""+Integer.MAX_VALUE) int limit,
- @QueryParam("page") @DefaultValue("0") int page);
+ @QueryParam("page") @DefaultValue("0") int page,
+ @QueryParam("include") @Separator(",") List<String> include,
+ @QueryParam("exclude") @Separator(",") List<String> exclude);
 
  @POST
  @Consumes(MediaType.APPLICATION_JSON)

horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java

@@ -271,10 +271,10 @@ public Object getData(int id, String token, String schemaUri) {
  }
  }
 
- //this is nearly identical to TestServiceImpl.listLabelValues (except the return object)
+ //this is nearly identical to TestServiceImpl.labelValues (except the return object)
  //this reads from the dataset table but provides data specific to the run...
  @Override
- public List<ExportedLabelValues> labelValues(int runId, String filter, String sort, String direction, int limit, int page){
+ public List<ExportedLabelValues> labelValues(int runId, String filter, String sort, String direction, int limit, int page, List<String> include, List<String> exclude){
  List<ExportedLabelValues> rtrn = new ArrayList<>();
  Run run = getRun(runId,null);
  if(run == null){
@@ -314,10 +314,25 @@ public List<ExportedLabelValues> labelValues(int runId, String filter, String so
  orderSql="order by combined.datasetId DESC";
  }
  }
+ String includeExcludeSql = "";
+ if (include!=null && !include.isEmpty()) {
+ if (exclude != null && !exclude.isEmpty()) {
+ include = new ArrayList<>(include);
+ include.removeAll(exclude);
+ }
+ if (!include.isEmpty()) {
+ includeExcludeSql = " AND label.name in :include";
+ }
+ }
+ //includeExcludeSql is empty if include did not contain entries after exclude removal
+ if(includeExcludeSql.isEmpty() && exclude!=null && !exclude.isEmpty()){
+ includeExcludeSql=" AND label.name NOT in :exclude";
+ }
+
  String sql = """
  WITH
  combined as (
- SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE label.name IS NOT NULL), '{}'::::jsonb) AS values, dataset.id AS datasetId, dataset.start AS start, dataset.stop AS stop
+ SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE label.name IS NOT NULL INCLUDE_EXCLUDE_PLACEHOLDER), '{}'::::jsonb) AS values, dataset.id AS datasetId, dataset.start AS start, dataset.stop AS stop
  FROM dataset
  LEFT JOIN label_values lv ON dataset.id = lv.dataset_id
  LEFT JOIN label ON label.id = lv.label_id
@@ -326,6 +341,7 @@ SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE la
  ) select * from combined FILTER_PLACEHOLDER ORDER_PLACEHOLDER limit :limit offset :offset
  """
  .replace("FILTER_PLACEHOLDER",filterSql)
+ .replace("INCLUDE_EXCLUDE_PLACEHOLDER",includeExcludeSql)
  .replace("ORDER_PLACEHOLDER",orderSql);
 
  NativeQuery query = ((NativeQuery) em.createNativeQuery(sql))
@@ -337,12 +353,17 @@ SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE la
  query.setParameter("filter", filter);
  }
  }
+ if(includeExcludeSql.contains(":include")){
+ query.setParameter("include",include);
+ }else if (includeExcludeSql.contains(":exclude")){
+ query.setParameter("exclude",exclude);
+ }
  if(orderSql.contains(":orderBy")){
  query.setParameter("orderBy",sort);
  }
  query
  .setParameter("limit",limit)
- .setParameter("offset",limit * page)
+ .setParameter("offset",limit * Math.max(0,page))
  .unwrap(NativeQuery.class)
  .addScalar("values", JsonBinaryType.INSTANCE)
  .addScalar("datasetId",Integer.class)

horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/TestServiceImpl.java

@@ -78,7 +78,7 @@ public class TestServiceImpl implements TestService {
  protected static final String LABEL_VALUES_QUERY = """
  WITH
  combined as (
- SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE label.name IS NOT NULL), '{}'::::jsonb) AS values, runId, dataset.id AS datasetId, dataset.start AS start, dataset.stop AS stop
+ SELECT DISTINCT COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE label.name IS NOT NULL INCLUDE_EXCLUDE_PLACEHOLDER), '{}'::::jsonb) AS values, runId, dataset.id AS datasetId, dataset.start AS start, dataset.stop AS stop
  FROM dataset
  LEFT JOIN label_values lv ON dataset.id = lv.dataset_id
  LEFT JOIN label ON label.id = lv.label_id
@@ -550,9 +550,10 @@ private boolean isPredicate(String input){
  return false;
  }
 
+ @Transactional
  @WithRoles
  @Override
- public List<ExportedLabelValues> listLabelValues(int testId, String filter, String before, String after, boolean filtering, boolean metrics, String sort, String direction, int limit, int page) {
+ public List<ExportedLabelValues> labelValues(int testId, String filter, String before, String after, boolean filtering, boolean metrics, String sort, String direction, int limit, int page, List<String> include, List<String> exclude) {
  Test test = get(testId,null);
  if(test == null){
  throw ServiceException.serverError("Cannot find test "+testId);
@@ -600,6 +601,22 @@ public List<ExportedLabelValues> listLabelValues(int testId, String filter, Stri
  filterSql+=FILTER_SEPARATOR+FILTER_AFTER;
  }
  }
+
+ String includeExcludeSql = "";
+ if (include!=null && !include.isEmpty()){
+ if(exclude!=null && !exclude.isEmpty()){
+ include = new ArrayList<>(include);
+ include.removeAll(exclude);
+ }
+ if(!include.isEmpty()) {
+ includeExcludeSql = " AND label.name in :include";
+ }
+ }
+ //includeExcludeSql is empty if include did not contain entries after exclude removal
+ if(includeExcludeSql.isEmpty() && exclude!=null && !exclude.isEmpty()){
+ includeExcludeSql=" AND label.name NOT in :exclude";
+ }
+
  if(filterSql.isBlank() && filter != null && !filter.isBlank()){
  //TODO there was an error with the filter, do we return that info to the user?
  }
@@ -625,7 +642,9 @@ public List<ExportedLabelValues> listLabelValues(int testId, String filter, Stri
 
  String sql = LABEL_VALUES_QUERY
  .replace("FILTER_PLACEHOLDER",filterSql)
+ .replace("INCLUDE_EXCLUDE_PLACEHOLDER",includeExcludeSql)
  .replace("ORDER_PLACEHOLDER",orderSql);
+
  NativeQuery query = ((NativeQuery) em.createNativeQuery(sql))
  .setParameter("testId", test.id)
  .setParameter("filteringLabels", filtering)
@@ -644,12 +663,17 @@ public List<ExportedLabelValues> listLabelValues(int testId, String filter, Stri
  query.setParameter("after",afterInstant, StandardBasicTypes.INSTANT);
  }
  }
+ if(includeExcludeSql.contains(":include")){
+ query.setParameter("include",include);
+ }else if (includeExcludeSql.contains(":exclude")){
+ query.setParameter("exclude",exclude);
+ }
  if(orderSql.contains(LABEL_ORDER_JSONPATH)){
  query.setParameter("orderBy", sort);
  }
-  query
- .setParameter("limit",limit)//limit
- .setParameter("offset",limit * page)//offset
+ query
+ .setParameter("limit",limit)
+ .setParameter("offset",limit * Math.max(0,page))
  .unwrap(NativeQuery.class)
  .addScalar("values", JsonBinaryType.INSTANCE)
  .addScalar("runId",Integer.class)