Skip to content

Commit

Permalink
[ALS-4412] - Implement query ID API endpoints (#111)
Browse files Browse the repository at this point in the history 
* Refactoring AggDataSharingResourceRS

- Make logging more consistent
- Check for 2xx response codes, not just 200
- Deduplicate constructor code
- Make some constants final

* [ALS-4412] - Implement query ID API endpoints

- Implement proxy query, queryStatus, and queryResult methods
- Deduplicate code
- Fix 2xx predicates

* Unit tests + bug fixes

- Added coverage to AggregateDataSharingResourceRS
- Made a helper class for those tests
- Fixed a marshalling bug

* Do not persist to common area repo
  • Loading branch information
@Luke-Sikina
Luke-Sikina committed Jul 26, 2023
1 parent d09793f commit 807cb3f
Show file tree
Hide file tree
Showing 4 changed files with 283 additions and 112 deletions.
191 changes: 92 additions & 99 deletions ...-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
package edu.harvard.hms.dbmi.avillach;

import static edu.harvard.dbmi.avillach.service.ResourceWebClient.QUERY_METADATA_FIELD;
import static edu.harvard.dbmi.avillach.util.HttpClientUtil.composeURL;
import static edu.harvard.dbmi.avillach.util.HttpClientUtil.readObjectFromResponse;
import static edu.harvard.dbmi.avillach.util.HttpClientUtil.retrievePostResponse;
import static edu.harvard.dbmi.avillach.util.HttpClientUtil.throwResponseError;

import java.io.IOException;
import java.util.*;
Expand All @@ -18,6 +15,7 @@
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;

import edu.harvard.dbmi.avillach.util.HttpClientUtil;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
Expand Down Expand Up @@ -46,42 +44,30 @@ public class AggregateDataSharingResourceRS implements IResourceRS {

private static final ObjectMapper objectMapper = new ObjectMapper();

private Header[] headers;
private final Header[] headers;

private static final String BEARER_STRING = "Bearer ";

private final static ObjectMapper json = new ObjectMapper();
private Logger logger = LoggerFactory.getLogger(this.getClass());
private final Logger logger = LoggerFactory.getLogger(this.getClass());

private final int threshold;
private final int variance;

private final String randomSalt;

public static final List<String> ALLOWED_RESULT_TYPES = Arrays.asList(new String [] {
"COUNT", "CROSS_COUNT", "INFO_COLUMN_LISTING", "OBSERVATION_COUNT", "OBSERVATION_CROSS_COUNT"
});

public AggregateDataSharingResourceRS() {
logger.info("initialize Aggregate Resource NO INJECTION");

if (properties == null) {
properties = new ApplicationProperties();
properties.init("pic-sure-aggregate-resource");
}

threshold = properties.getTargetPicsureObfuscationThreshold();
variance = properties.getTargetPicsureObfuscationVariance();
randomSalt = properties.getTargetPicsureObfuscationSalt();

headers = new Header[] {new BasicHeader(HttpHeaders.AUTHORIZATION, BEARER_STRING + properties.getTargetPicsureToken())};

this(null);
}

@Inject
public AggregateDataSharingResourceRS(ApplicationProperties applicationProperties) {
this.properties = applicationProperties;
logger.info("initialize Aggregate Resource Injected " + applicationProperties);
if (applicationProperties == null) {
logger.info("initialize Aggregate Resource NO INJECTION");
} else {
logger.info("initialize Aggregate Resource Injected " + applicationProperties);
}

if (properties == null) {
properties = new ApplicationProperties();
Expand Down Expand Up @@ -124,12 +110,12 @@ public ResourceInfo info(QueryRequest infoRequest) {
}

String payload = objectMapper.writeValueAsString(chainRequest);

HttpResponse response = retrievePostResponse(composeURL(properties.getTargetPicsureUrl(), pathName), headers, payload);
if (response.getStatusLine().getStatusCode() != 200) {
String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName);
HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, payload);
if (!HttpClientUtil.is2xx(response)) {
logger.error("{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName,
response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase());
throwResponseError(response, properties.getTargetPicsureUrl());
HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl());
}

//if we are proxying an info request, we need to return our own resource ID
Expand All @@ -152,77 +138,74 @@ public ResourceInfo info(QueryRequest infoRequest) {
@Override
public SearchResults search(QueryRequest searchRequest) {
logger.debug("Calling Aggregate Data Sharing Search");
if (searchRequest == null) {
throw new ProtocolException(ProtocolException.MISSING_DATA);
}
Object search = searchRequest.getQuery();
if (search == null) {
throw new ProtocolException((ProtocolException.MISSING_DATA));
}

String pathName = "/search";
try {
QueryRequest chainRequest = new QueryRequest();
chainRequest.setQuery(searchRequest.getQuery());
chainRequest.setResourceCredentials(searchRequest.getResourceCredentials());

if(properties.getTargetResourceId() != null && !properties.getTargetResourceId().isEmpty()) {
chainRequest.setResourceUUID(UUID.fromString(properties.getTargetResourceId()));
} else {
chainRequest.setResourceUUID(searchRequest.getResourceUUID());
}

String payload = objectMapper.writeValueAsString(chainRequest);

HttpResponse response = retrievePostResponse(composeURL(properties.getTargetPicsureUrl(), pathName), headers, payload);
if (response.getStatusLine().getStatusCode() != 200) {
logger.error("{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName,
response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase());
throwResponseError(response, properties.getTargetPicsureUrl());
}
return readObjectFromResponse(response, SearchResults.class);
} catch (IOException e) {
// Note: this shouldn't ever happen
logger.error("Error encoding search payload", e);
throw new ApplicationException(
"Error encoding search for resource with id " + searchRequest.getResourceUUID());
}
checkQuery(searchRequest);
HttpResponse response = postRequest(searchRequest, "/search");
return readObjectFromResponse(response, SearchResults.class);
}

@POST
@Path("/query")
@Override
public QueryStatus query(QueryRequest queryJson) {
public QueryStatus query(QueryRequest queryRequest) {
logger.debug("Calling Aggregate Data Sharing Resource query()");
throw new UnsupportedOperationException("Query is not implemented in this resource. Please use query/sync");
checkQuery(queryRequest);
HttpResponse response = postRequest(queryRequest, "/query");
return readObjectFromResponse(response, QueryStatus.class);

}

@POST
@Path("/query/{resourceQueryId}/status")
@Override
public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest statusQuery) {
public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest statusRequest) {
logger.debug("Calling Aggregate Data Sharing Resource queryStatus() for query {}", queryId);
throw new UnsupportedOperationException(
"Query status is not implemented in this resource. Please use query/sync");
checkQuery(statusRequest);
HttpResponse response = postRequest(statusRequest, "/query/" + queryId + "/status");
return readObjectFromResponse(response, QueryStatus.class);
}

@POST
@Path("/query/{resourceQueryId}/result")
@Override
public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryRequest resultRequest) {
logger.debug("Calling Aggregate Data Sharing Resource queryResult() for query {}", queryId);
throw new UnsupportedOperationException(
"Query result is not implemented in this resource. Please use query/sync");
checkQuery(resultRequest);
HttpResponse response = postRequest(resultRequest, "/query/" + queryId + "/result");
try {
return Response.ok(response.getEntity().getContent()).build();
} catch (IOException e) {
throw new ApplicationException(
"Error encoding query for resource with id " + resultRequest.getResourceUUID()
);
}
}

private HttpResponse postRequest(QueryRequest statusRequest, String pathName) {
try {
QueryRequest chainRequest = createChainRequest(statusRequest);
String payload = objectMapper.writeValueAsString(chainRequest);
String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName);
HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, payload);
if (!HttpClientUtil.is2xx(response)) {
logger.error("{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName,
response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase());
HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl());
}
return response;
} catch (IOException e) {
// Note: this shouldn't ever happen
logger.error("Error encoding search payload", e);
throw new ApplicationException(
"Error encoding search for resource with id " + statusRequest.getResourceUUID());
}
}

@POST
@Path("/query/sync")
@Override
public Response querySync(QueryRequest queryRequest) {
logger.debug("Calling Aggregate Data Sharing Resource querySync()");
if (queryRequest == null || queryRequest.getQuery() == null) {
throw new ProtocolException(ProtocolException.MISSING_DATA);
}
checkQuery(queryRequest);

try {
Object query = queryRequest.getQuery();
Expand All @@ -234,35 +217,34 @@ public Response querySync(QueryRequest queryRequest) {
}
String expectedResultType = jsonNode.get("expectedResultType").asText();

if (! ALLOWED_RESULT_TYPES.contains(expectedResultType)) {
Set<String> allowedResultTypes =
Set.of("COUNT", "CROSS_COUNT", "INFO_COLUMN_LISTING", "OBSERVATION_COUNT", "OBSERVATION_CROSS_COUNT");
if (!allowedResultTypes.contains(expectedResultType)) {
logger.warn("Incorrect Result Type: " + expectedResultType);
return Response.status(Response.Status.BAD_REQUEST).build();
}

String targetPicsureUrl = properties.getTargetPicsureUrl();
String queryString = json.writeValueAsString(queryRequest);
String payload = json.writeValueAsString(queryRequest);
String pathName = "/query/sync";
String composedURL = composeURL(targetPicsureUrl, pathName);
logger.debug("Aggregate Data Sharing Resource, sending query: " + queryString + ", to: " + composedURL);
HttpResponse response = retrievePostResponse(composedURL, headers, queryString);
if (response.getStatusLine().getStatusCode() != 200) {
logger.error("Not 200 status!");
String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName);
logger.debug("Aggregate Data Sharing Resource, sending query: " + payload + ", to: " + composedURL);
HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, payload);
if (!HttpClientUtil.is2xx(response)) {
logger.error(
composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ",
"{} calling resource with id {} did not return a 200: {} {} ", composedURL, resourceUUID,
response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase());
throwResponseError(response, targetPicsureUrl);
HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl());
}


HttpEntity entity = response.getEntity();
String entityString = EntityUtils.toString(entity, "UTF-8");
String responseString = entityString;

if(expectedResultType.equals("COUNT")) {
if("COUNT".equals(expectedResultType)) {
responseString = aggregateCount(entityString).orElse(entityString);
} else if(expectedResultType.equals("CROSS_COUNT")) {
} else if("CROSS_COUNT".equals(expectedResultType)) {
Map<String, String> crossCounts = processCrossCounts(entityString);

responseString = objectMapper.writeValueAsString(crossCounts);
}

Expand All @@ -287,28 +269,20 @@ public Response querySync(QueryRequest queryRequest) {
@POST
@Path("/query/format")
public Response queryFormat(QueryRequest queryRequest) {
if (queryRequest == null) {
throw new ProtocolException(ProtocolException.MISSING_DATA);
}
Object search = queryRequest.getQuery();
if (search == null) {
throw new ProtocolException((ProtocolException.MISSING_DATA));
}
checkQuery(queryRequest);

UUID resourceUUID = queryRequest.getResourceUUID();
String pathName = "/query/format";

try {
String targetPicsureUrl = properties.getTargetPicsureUrl();
String queryString = json.writeValueAsString(queryRequest);
String composedURL = composeURL(targetPicsureUrl, pathName);
HttpResponse response = retrievePostResponse(composeURL(properties.getTargetPicsureUrl(), pathName), headers, queryString);
if (response.getStatusLine().getStatusCode() != 200) {
logger.error("Not 200 status!");
String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName);
HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, queryString);
if (!HttpClientUtil.is2xx(response)) {
logger.error(
composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ",
response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase());
throwResponseError(response, targetPicsureUrl);
HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl());
}

return Response.ok(response.getEntity().getContent()).build();
Expand All @@ -323,7 +297,7 @@ public Response queryFormat(QueryRequest queryRequest) {

private Map<String, String> processCrossCounts(String entityString) throws com.fasterxml.jackson.core.JsonProcessingException {
Map<String, String> crossCounts = objectMapper.readValue(entityString, new TypeReference<>(){});
final List<Map.Entry<String, String>> entryList = new ArrayList(crossCounts.entrySet());
final List<Map.Entry<String, String>> entryList = new ArrayList<>(crossCounts.entrySet());
entryList.sort(Map.Entry.comparingByKey());
final StringBuffer crossCountsString = new StringBuffer();
entryList.forEach(entry -> crossCountsString.append(entry.getKey()).append(":").append(entry.getValue()).append("\n"));
Expand All @@ -350,6 +324,25 @@ private Map<String, String> processCrossCounts(String entityString) throws com.f
return crossCounts;
}

private QueryRequest createChainRequest(QueryRequest queryRequest) {
QueryRequest chainRequest = new QueryRequest();
chainRequest.setQuery(queryRequest.getQuery());
chainRequest.setResourceCredentials(queryRequest.getResourceCredentials());

if(properties.getTargetResourceId() != null && !properties.getTargetResourceId().isEmpty()) {
chainRequest.setResourceUUID(UUID.fromString(properties.getTargetResourceId()));
} else {
chainRequest.setResourceUUID(queryRequest.getResourceUUID());
}
return chainRequest;
}

private static void checkQuery(QueryRequest searchRequest) {
if (searchRequest == null || searchRequest.getQuery() == null) {
throw new ProtocolException(ProtocolException.MISSING_DATA);
}
}

private int generateRequestVariance(String entityString) {
return Math.abs((entityString + randomSalt).hashCode()) % (variance * 2 + 1) - variance;
}
Expand Down
Loading

0 comments on commit 807cb3f

Please sign in to comment.