Enrich Jobs endpoint with filtering functionality

yahoo · Sep 12, 2016 · de1de63 · de1de63
1 parent 3472549
commit de1de63
Show file tree

Hide file tree

Showing 9 changed files with 232 additions and 93 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,8 @@ Current
 -------
 
 ### Added:
+- [Enrich jobs endpoint with filtering functionality] (https://github.com/yahoo/fili/pull/26)
+   * Jobs endpoint now supports filters
 
 -  [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34)
     * Added two tests `LookupDimensionFilteringDataServeletSpec` and `LookupDimensionGroupingDataServletSpec` to test QTL functionality
@@ -34,6 +36,9 @@ Current
 
 ### Changed:
 
+- [Enrich jobs endpoint with filtering functionality] (https://github.com/yahoo/fili/pull/26)
+  * The default job payload generated by `DefaultJobPayloadBuilder` now has userId meta data
+
 - [Memoize generated values during recursive class-scan class construction](https://github.com/yahoo/fili/pull/29)
 
 -  [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34)

diff --git a/...src/main/java/com/yahoo/bard/webservice/async/jobs/payloads/DefaultJobPayloadBuilder.java b/...src/main/java/com/yahoo/bard/webservice/async/jobs/payloads/DefaultJobPayloadBuilder.java
@@ -40,6 +40,7 @@ public Map<String, String> buildPayload(JobRow jobRow, UriInfo uriInfo) {
         job.put("status", fieldValueMap.get(DefaultJobField.STATUS.getName()));
         job.put("jobTicket", fieldValueMap.get(DefaultJobField.JOB_TICKET.getName()));
         job.put("dateCreated", fieldValueMap.get(DefaultJobField.DATE_CREATED.getName()));
+        job.put("userId", fieldValueMap.get(DefaultJobField.USER_ID.getName()));
 
         //throw exception if any of the JobFields are missing in the map
         if (job.containsValue(null)) {

diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/JobsApiRequest.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/JobsApiRequest.java
@@ -6,6 +6,7 @@
 import com.yahoo.bard.webservice.async.broadcastchannels.BroadcastChannel;
 import com.yahoo.bard.webservice.async.jobs.payloads.JobPayloadBuilder;
 import com.yahoo.bard.webservice.async.jobs.jobrows.JobRow;
+import com.yahoo.bard.webservice.async.jobs.stores.JobRowFilter;
 import com.yahoo.bard.webservice.async.preresponses.stores.PreResponseStore;
 
 import org.slf4j.Logger;
@@ -14,8 +15,11 @@
 import rx.Observable;
 import rx.observables.ConnectableObservable;
 
+import java.util.Arrays;
+import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import javax.validation.constraints.NotNull;
 import javax.ws.rs.core.UriInfo;
@@ -32,6 +36,7 @@ public class JobsApiRequest extends ApiRequest {
     private final ApiJobStore apiJobStore;
     private final PreResponseStore preResponseStore;
     private final BroadcastChannel<String> broadcastChannel;
+    private final String filters;
 
     /**
      * Parses the API request URL and generates the Api Request object.
@@ -42,6 +47,10 @@ public class JobsApiRequest extends ApiRequest {
      * must be a positive integer. If not present, must be the empty string.
      * @param page  desired page of results. If present in the original request, must be a positive
      * integer. If not present, must be the empty string.
+     * @param filters  URL filter query String in the format:
+     * <pre>{@code
+     * ((field name and operation):((multiple values bounded by [])or(single value))))(followed by , or end of string)
+     * }</pre>
      * @param uriInfo  The URI of the request object.
      * @param jobPayloadBuilder  The JobRowMapper to be used to map JobRow to the Job returned by the api
      * @param apiJobStore  The ApiJobStore containing Job metadata
@@ -54,6 +63,7 @@ public JobsApiRequest(
             String asyncAfter,
             @NotNull String perPage,
             @NotNull String page,
+            String filters,
             UriInfo uriInfo,
             JobPayloadBuilder jobPayloadBuilder,
             ApiJobStore apiJobStore,
@@ -65,6 +75,7 @@ public JobsApiRequest(
         this.apiJobStore = apiJobStore;
         this.preResponseStore = preResponseStore;
         this.broadcastChannel = broadcastChannel;
+        this.filters = filters;
     }
 
     /**
@@ -112,15 +123,44 @@ public Observable<PreResponse> handleBroadcastChannelNotification(@NotNull Strin
     }
 
     /**
-     * Returns an Observable containing a stream of job payloads for all the jobs in the ApiJobStore. If, for any
-     * JobRow, the mapping from JobRow to job view fails, an Observable over JobRequestFailedException is returned. If
-     * the ApiJobStore is empty, we return an empty Observable.
+     * Return an Observable containing a stream of job views for jobs in the ApiJobStore. If filter String is non null
+     * and non empty, only return results that satisfy the filter. If filter String is null or empty, return all rows.
+     * If, for any JobRow, the mapping from JobRow to job view fails, an Observable over JobRequestFailedException is
+     * returned. If the ApiJobStore is empty, we return an empty Observable.
      *
      * @return An Observable containing a stream of Maps representing the job to be returned to the user
      */
     public Observable<Map<String, String>> getJobViews() {
-        return apiJobStore.getAllRows()
-                .map(this::mapJobRowsToJobViews);
+        if (filters == null || "".equals(filters)) {
+            return apiJobStore.getAllRows().map(this::mapJobRowsToJobViews);
+        } else {
+            return apiJobStore.getFilteredRows(buildJobStoreFilter(filters)).map(this::mapJobRowsToJobViews);
+        }
+    }
+
+    /**
+     * Given a filter String, generates a Set of ApiJobStoreFilters. This method will throw a BadApiRequestException if
+     * the filter String cannot be parsed into ApiJobStoreFilters successfully.
+     *
+     * @param filterQuery  Expects a URL filterQuery String that may contain multiple filters separated by
+     * comma.  The format of a filter String is :
+     * (JobField name)-(operation)[(value or comma separated values)]?
+     *
+     * @return  A Set of ApiJobStoreFilters
+     */
+    public LinkedHashSet<JobRowFilter> buildJobStoreFilter(@NotNull String filterQuery) {
+        // split on '],' to get list of filters
+        return Arrays.stream(filterQuery.split(COMMA_AFTER_BRACKET_PATTERN))
+                .map(
+                        filter -> {
+                            try {
+                                return new JobRowFilter(filter);
+                            } catch (BadFilterException e) {
+                                throw new BadApiRequestException(e.getMessage(), e);
+                            }
+                        }
+                )
+                .collect(Collectors.toCollection(LinkedHashSet<JobRowFilter>::new));
     }
 
     /**
@@ -134,7 +174,7 @@ public Observable<Map<String, String>> getJobViews() {
     private Map<String, String> mapJobRowsToJobViews(JobRow jobRow) {
         try {
             return jobPayloadBuilder.buildPayload(jobRow, uriInfo);
-        } catch (JobRequestFailedException e) {
+        } catch (JobRequestFailedException ignored) {
             String msg = ErrorMessageFormat.JOBS_RETREIVAL_FAILED.format(jobRow.getId());
             LOG.error(msg);
             throw new JobRequestFailedException(msg);

diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/JobsServlet.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/JobsServlet.java
@@ -8,8 +8,6 @@
 import static javax.ws.rs.core.Response.Status.OK;
 
 import com.yahoo.bard.webservice.application.ObjectMappersSuite;
-import com.yahoo.bard.webservice.config.SystemConfig;
-import com.yahoo.bard.webservice.config.SystemConfigProvider;
 import com.yahoo.bard.webservice.data.HttpResponseChannel;
 import com.yahoo.bard.webservice.data.HttpResponseMaker;
 import com.yahoo.bard.webservice.data.dimension.DimensionDictionary;
@@ -71,7 +69,6 @@
 public class JobsServlet extends EndpointServlet {
 
     private static final Logger LOG = LoggerFactory.getLogger(JobsServlet.class);
-    private static final SystemConfig SYSTEM_CONFIG = SystemConfigProvider.getInstance();
 
     private final ApiJobStore apiJobStore;
     private final RequestMapper requestMapper;
@@ -118,6 +115,9 @@ public JobsServlet(
      * @param perPage  Requested number of rows of data to be displayed on each page of results
      * @param page  Requested page of results desired
      * @param format  Requested format
+     * @param filters  Filters to be applied on the JobRows. Expects a URL filter query String that may contain multiple
+     * filter strings separated by comma.  The format of a filter String is :
+     * (JobField name)-(operation)[(value or comma separated values)]?
      * @param uriInfo  UriInfo of the request
      * @param containerRequestContext  The context of data provided by the Jersey container for this request
      * @param asyncResponse  An asyncAfter response that we can use to respond asynchronously
@@ -128,6 +128,7 @@ public void getJobs(
             @DefaultValue("") @NotNull @QueryParam("perPage") String perPage,
             @DefaultValue("") @NotNull @QueryParam("page") String page,
             @QueryParam("format") String format,
+            @QueryParam("filters") String filters,
             @Context UriInfo uriInfo,
             @Context ContainerRequestContext containerRequestContext,
             @Suspended AsyncResponse asyncResponse
@@ -138,9 +139,10 @@ public void getJobs(
 
             JobsApiRequest apiRequest = new JobsApiRequest (
                     format,
-                    null, //asynAfter is null so it behaves like a synchronous request
+                    null, //asyncAfter is null so it behaves like a synchronous request
                     perPage,
                     page,
+                    filters,
                     uriInfo,
                     jobPayloadBuilder,
                     apiJobStore,
@@ -164,8 +166,7 @@ public void getJobs(
                                     )
                     );
 
-            apiRequest.getJobViews()
-                    .toList()
+            apiRequest.getJobViews().toList()
                     .map(jobs -> jobsApiRequest.getPage(paginationFactory.apply(jobs)))
                     .map(result -> formatResponse(jobsApiRequest, result, "jobs", null))
                     .defaultIfEmpty(getResponse("{}"))
@@ -213,6 +214,7 @@ public void getJobByTicket(
                     null,
                     "",
                     "",
+                    null, //filter string is null
                     uriInfo,
                     jobPayloadBuilder,
                     apiJobStore,
@@ -272,6 +274,7 @@ public void getJobResultsByTicket(
                     asyncAfter,
                     perPage,
                     page,
+                    null, // filter string is null
                     uriInfo,
                     jobPayloadBuilder,
                     apiJobStore,

diff --git a/.../groovy/com/yahoo/bard/webservice/async/jobs/payloads/DefaultJobPayloadBuilderSpec.groovy b/.../groovy/com/yahoo/bard/webservice/async/jobs/payloads/DefaultJobPayloadBuilderSpec.groovy
@@ -47,7 +47,8 @@ class DefaultJobPayloadBuilderSpec extends Specification {
                 self: "https://localhost:9998/v1/jobs/ticket1",
                 status: "success",
                 jobTicket: "ticket1",
-                dateCreated: "2016-01-01"
+                dateCreated: "2016-01-01",
+                userId: "momo"
         ]
 
         expect:

diff --git a/...d/webservice/jobs/JobRowFilterSpec.groovy → ...async/jobs/stores/JobRowFilterSpec.groovy b/...d/webservice/jobs/JobRowFilterSpec.groovy → ...async/jobs/stores/JobRowFilterSpec.groovy
@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc.
 // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
-package com.yahoo.bard.webservice.jobs
+package com.yahoo.bard.webservice.async.jobs.stores
 
 import com.yahoo.bard.webservice.async.jobs.stores.JobRowFilter
 import com.yahoo.bard.webservice.web.BadFilterException